Spaces:
Running
Running
Update README.md
#2
by
Realcat
- opened
- README.md +2 -2
- api/__init__.py +0 -0
- api/client.py +148 -70
- api/server.py +393 -29
- api/test/CMakeLists.txt +16 -0
- api/test/build_and_run.sh +16 -0
- api/test/client.cpp +84 -0
- api/test/helper.h +410 -0
- api/types.py +16 -0
- requirements.txt +1 -2
- test_app_cli.py +9 -8
- ui/__init__.py +5 -0
- ui/api.py +0 -293
- ui/app_class.py +4 -20
- ui/config.yaml +6 -2
- ui/sfm.py +8 -2
- ui/utils.py +4 -2
- ui/viz.py +3 -0
README.md
CHANGED
@@ -4,7 +4,7 @@ emoji: 🤗
|
|
4 |
colorFrom: red
|
5 |
colorTo: yellow
|
6 |
sdk: gradio
|
7 |
-
sdk_version:
|
8 |
app_file: app.py
|
9 |
pinned: true
|
10 |
license: apache-2.0
|
@@ -152,4 +152,4 @@ This code is built based on [Hierarchical-Localization](https://github.com/cvg/H
|
|
152 |
[stars-shield]: https://img.shields.io/github/stars/Vincentqyw/image-matching-webui.svg?style=for-the-badge
|
153 |
[stars-url]: https://github.com/Vincentqyw/image-matching-webui/stargazers
|
154 |
[issues-shield]: https://img.shields.io/github/issues/Vincentqyw/image-matching-webui.svg?style=for-the-badge
|
155 |
-
[issues-url]: https://github.com/Vincentqyw/image-matching-webui/issues
|
|
|
4 |
colorFrom: red
|
5 |
colorTo: yellow
|
6 |
sdk: gradio
|
7 |
+
sdk_version: 5.3.0
|
8 |
app_file: app.py
|
9 |
pinned: true
|
10 |
license: apache-2.0
|
|
|
152 |
[stars-shield]: https://img.shields.io/github/stars/Vincentqyw/image-matching-webui.svg?style=for-the-badge
|
153 |
[stars-url]: https://github.com/Vincentqyw/image-matching-webui/stargazers
|
154 |
[issues-shield]: https://img.shields.io/github/issues/Vincentqyw/image-matching-webui.svg?style=for-the-badge
|
155 |
+
[issues-url]: https://github.com/Vincentqyw/image-matching-webui/issues
|
api/__init__.py
ADDED
File without changes
|
api/client.py
CHANGED
@@ -1,18 +1,102 @@
|
|
1 |
import argparse
|
|
|
|
|
2 |
import pickle
|
3 |
import time
|
4 |
-
from typing import Dict
|
5 |
|
|
|
6 |
import numpy as np
|
7 |
import requests
|
8 |
-
from loguru import logger
|
9 |
|
10 |
-
|
11 |
-
|
12 |
-
|
13 |
|
|
|
14 |
|
15 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
16 |
"""
|
17 |
Send a request to the API to generate a match between two images.
|
18 |
|
@@ -28,6 +112,7 @@ def send_generate_request(path0: str, path1: str) -> Dict[str, np.ndarray]:
|
|
28 |
"""
|
29 |
files = {"image0": open(path0, "rb"), "image1": open(path1, "rb")}
|
30 |
try:
|
|
|
31 |
response = requests.post(API_URL_MATCH, files=files)
|
32 |
pred = {}
|
33 |
if response.status_code == 200:
|
@@ -44,68 +129,56 @@ def send_generate_request(path0: str, path1: str) -> Dict[str, np.ndarray]:
|
|
44 |
return pred
|
45 |
|
46 |
|
47 |
-
def
|
|
|
|
|
48 |
"""
|
49 |
Send a request to the API to extract features from an image.
|
50 |
|
51 |
Args:
|
52 |
-
|
53 |
|
54 |
Returns:
|
55 |
-
Dict[str, np.ndarray]: A
|
56 |
-
The keys are "keypoints", "descriptors", and
|
57 |
-
values are ndarrays of shape (N, 2), (N, 128),
|
58 |
-
respectively.
|
59 |
"""
|
60 |
-
|
61 |
-
|
62 |
-
|
63 |
-
|
64 |
-
|
65 |
-
|
66 |
-
|
67 |
-
|
68 |
-
|
69 |
-
|
70 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
71 |
)
|
72 |
-
|
73 |
-
files["image"].close()
|
74 |
-
return pred
|
75 |
-
|
76 |
|
77 |
-
def send_generate_request2(image_path: str) -> Dict[str, np.ndarray]:
|
78 |
-
"""
|
79 |
-
Send a request to the API to extract features from an image.
|
80 |
-
|
81 |
-
Args:
|
82 |
-
image_path (str): The path to the image.
|
83 |
|
84 |
-
|
85 |
-
Dict[str, np.ndarray]: A dictionary containing the extracted features.
|
86 |
-
The keys are "keypoints", "descriptors", and "scores", and the
|
87 |
-
values are ndarrays of shape (N, 2), (N, 128), and (N,), respectively.
|
88 |
-
"""
|
89 |
-
data = {
|
90 |
-
"image_path": image_path,
|
91 |
-
"max_keypoints": 1024,
|
92 |
-
"reference_points": [[0.0, 0.0], [1.0, 1.0]],
|
93 |
-
}
|
94 |
-
pred = {}
|
95 |
try:
|
96 |
-
response = requests.
|
97 |
-
|
98 |
-
if response.status_code == 200:
|
99 |
-
pred = response.json()
|
100 |
-
for key in list(pred.keys()):
|
101 |
-
pred[key] = np.array(pred[key])
|
102 |
-
else:
|
103 |
-
print(
|
104 |
-
f"Error: Response code {response.status_code} - {response.text}"
|
105 |
-
)
|
106 |
except Exception as e:
|
107 |
print(f"An error occurred: {e}")
|
108 |
-
return pred
|
109 |
|
110 |
|
111 |
if __name__ == "__main__":
|
@@ -116,32 +189,37 @@ if __name__ == "__main__":
|
|
116 |
"--image0",
|
117 |
required=False,
|
118 |
help="Path for the file's melody",
|
119 |
-
default="
|
120 |
)
|
121 |
parser.add_argument(
|
122 |
"--image1",
|
123 |
required=False,
|
124 |
help="Path for the file's melody",
|
125 |
-
default="
|
126 |
)
|
127 |
args = parser.parse_args()
|
128 |
-
for i in range(10):
|
129 |
-
t1 = time.time()
|
130 |
-
preds = send_generate_request(args.image0, args.image1)
|
131 |
-
t2 = time.time()
|
132 |
-
logger.info(f"Time cost1: {(t2 - t1)} seconds")
|
133 |
-
|
134 |
-
for i in range(10):
|
135 |
-
t1 = time.time()
|
136 |
-
preds = send_generate_request1(args.image0)
|
137 |
-
t2 = time.time()
|
138 |
-
logger.info(f"Time cost2: {(t2 - t1)} seconds")
|
139 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
140 |
for i in range(10):
|
141 |
t1 = time.time()
|
142 |
-
preds =
|
143 |
t2 = time.time()
|
144 |
-
|
145 |
|
|
|
146 |
with open("preds.pkl", "wb") as f:
|
147 |
pickle.dump(preds, f)
|
|
|
1 |
import argparse
|
2 |
+
import base64
|
3 |
+
import os
|
4 |
import pickle
|
5 |
import time
|
6 |
+
from typing import Dict, List
|
7 |
|
8 |
+
import cv2
|
9 |
import numpy as np
|
10 |
import requests
|
|
|
11 |
|
12 |
+
ENDPOINT = "http://127.0.0.1:8001"
|
13 |
+
if "REMOTE_URL_RAILWAY" in os.environ:
|
14 |
+
ENDPOINT = os.environ["REMOTE_URL_RAILWAY"]
|
15 |
|
16 |
+
print(f"API ENDPOINT: {ENDPOINT}")
|
17 |
|
18 |
+
API_VERSION = f"{ENDPOINT}/version"
|
19 |
+
API_URL_MATCH = f"{ENDPOINT}/v1/match"
|
20 |
+
API_URL_EXTRACT = f"{ENDPOINT}/v1/extract"
|
21 |
+
|
22 |
+
|
23 |
+
def read_image(path: str) -> str:
|
24 |
+
"""
|
25 |
+
Read an image from a file, encode it as a JPEG and then as a base64 string.
|
26 |
+
|
27 |
+
Args:
|
28 |
+
path (str): The path to the image to read.
|
29 |
+
|
30 |
+
Returns:
|
31 |
+
str: The base64 encoded image.
|
32 |
+
"""
|
33 |
+
# Read the image from the file
|
34 |
+
img = cv2.imread(path, cv2.IMREAD_GRAYSCALE)
|
35 |
+
|
36 |
+
# Encode the image as a png, NO COMPRESSION!!!
|
37 |
+
retval, buffer = cv2.imencode(".png", img)
|
38 |
+
|
39 |
+
# Encode the JPEG as a base64 string
|
40 |
+
b64img = base64.b64encode(buffer).decode("utf-8")
|
41 |
+
|
42 |
+
return b64img
|
43 |
+
|
44 |
+
|
45 |
+
def do_api_requests(url=API_URL_EXTRACT, **kwargs):
|
46 |
+
"""
|
47 |
+
Helper function to send an API request to the image matching service.
|
48 |
+
|
49 |
+
Args:
|
50 |
+
url (str): The URL of the API endpoint to use. Defaults to the
|
51 |
+
feature extraction endpoint.
|
52 |
+
**kwargs: Additional keyword arguments to pass to the API.
|
53 |
+
|
54 |
+
Returns:
|
55 |
+
List[Dict[str, np.ndarray]]: A list of dictionaries containing the
|
56 |
+
extracted features. The keys are "keypoints", "descriptors", and
|
57 |
+
"scores", and the values are ndarrays of shape (N, 2), (N, ?),
|
58 |
+
and (N,), respectively.
|
59 |
+
"""
|
60 |
+
# Set up the request body
|
61 |
+
reqbody = {
|
62 |
+
# List of image data base64 encoded
|
63 |
+
"data": [],
|
64 |
+
# List of maximum number of keypoints to extract from each image
|
65 |
+
"max_keypoints": [100, 100],
|
66 |
+
# List of timestamps for each image (not used?)
|
67 |
+
"timestamps": ["0", "1"],
|
68 |
+
# Whether to convert the images to grayscale
|
69 |
+
"grayscale": 0,
|
70 |
+
# List of image height and width
|
71 |
+
"image_hw": [[640, 480], [320, 240]],
|
72 |
+
# Type of feature to extract
|
73 |
+
"feature_type": 0,
|
74 |
+
# List of rotation angles for each image
|
75 |
+
"rotates": [0.0, 0.0],
|
76 |
+
# List of scale factors for each image
|
77 |
+
"scales": [1.0, 1.0],
|
78 |
+
# List of reference points for each image (not used)
|
79 |
+
"reference_points": [[640, 480], [320, 240]],
|
80 |
+
# Whether to binarize the descriptors
|
81 |
+
"binarize": True,
|
82 |
+
}
|
83 |
+
# Update the request body with the additional keyword arguments
|
84 |
+
reqbody.update(kwargs)
|
85 |
+
try:
|
86 |
+
# Send the request
|
87 |
+
r = requests.post(url, json=reqbody)
|
88 |
+
if r.status_code == 200:
|
89 |
+
# Return the response
|
90 |
+
return r.json()
|
91 |
+
else:
|
92 |
+
# Print an error message if the response code is not 200
|
93 |
+
print(f"Error: Response code {r.status_code} - {r.text}")
|
94 |
+
except Exception as e:
|
95 |
+
# Print an error message if an exception occurs
|
96 |
+
print(f"An error occurred: {e}")
|
97 |
+
|
98 |
+
|
99 |
+
def send_request_match(path0: str, path1: str) -> Dict[str, np.ndarray]:
|
100 |
"""
|
101 |
Send a request to the API to generate a match between two images.
|
102 |
|
|
|
112 |
"""
|
113 |
files = {"image0": open(path0, "rb"), "image1": open(path1, "rb")}
|
114 |
try:
|
115 |
+
# TODO: replace files with post json
|
116 |
response = requests.post(API_URL_MATCH, files=files)
|
117 |
pred = {}
|
118 |
if response.status_code == 200:
|
|
|
129 |
return pred
|
130 |
|
131 |
|
132 |
+
def send_request_extract(
|
133 |
+
input_images: str, viz: bool = False
|
134 |
+
) -> List[Dict[str, np.ndarray]]:
|
135 |
"""
|
136 |
Send a request to the API to extract features from an image.
|
137 |
|
138 |
Args:
|
139 |
+
input_images (str): The path to the image.
|
140 |
|
141 |
Returns:
|
142 |
+
List[Dict[str, np.ndarray]]: A list of dictionaries containing the
|
143 |
+
extracted features. The keys are "keypoints", "descriptors", and
|
144 |
+
"scores", and the values are ndarrays of shape (N, 2), (N, 128),
|
145 |
+
and (N,), respectively.
|
146 |
"""
|
147 |
+
image_data = read_image(input_images)
|
148 |
+
inputs = {
|
149 |
+
"data": [image_data],
|
150 |
+
}
|
151 |
+
response = do_api_requests(
|
152 |
+
url=API_URL_EXTRACT,
|
153 |
+
**inputs,
|
154 |
+
)
|
155 |
+
print("Keypoints detected: {}".format(len(response[0]["keypoints"])))
|
156 |
+
|
157 |
+
# draw matching, debug only
|
158 |
+
if viz:
|
159 |
+
from hloc.utils.viz import plot_keypoints
|
160 |
+
from ui.viz import fig2im, plot_images
|
161 |
+
|
162 |
+
kpts = np.array(response[0]["keypoints_orig"])
|
163 |
+
if "image_orig" in response[0].keys():
|
164 |
+
img_orig = np.array(["image_orig"])
|
165 |
+
|
166 |
+
output_keypoints = plot_images([img_orig], titles="titles", dpi=300)
|
167 |
+
plot_keypoints([kpts])
|
168 |
+
output_keypoints = fig2im(output_keypoints)
|
169 |
+
cv2.imwrite(
|
170 |
+
"demo_match.jpg",
|
171 |
+
output_keypoints[:, :, ::-1].copy(), # RGB -> BGR
|
172 |
)
|
173 |
+
return response
|
|
|
|
|
|
|
174 |
|
|
|
|
|
|
|
|
|
|
|
|
|
175 |
|
176 |
+
def get_api_version():
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
177 |
try:
|
178 |
+
response = requests.get(API_VERSION).json()
|
179 |
+
print("API VERSION: {}".format(response["version"]))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
180 |
except Exception as e:
|
181 |
print(f"An error occurred: {e}")
|
|
|
182 |
|
183 |
|
184 |
if __name__ == "__main__":
|
|
|
189 |
"--image0",
|
190 |
required=False,
|
191 |
help="Path for the file's melody",
|
192 |
+
default="datasets/sacre_coeur/mapping_rot/02928139_3448003521_rot45.jpg",
|
193 |
)
|
194 |
parser.add_argument(
|
195 |
"--image1",
|
196 |
required=False,
|
197 |
help="Path for the file's melody",
|
198 |
+
default="datasets/sacre_coeur/mapping_rot/02928139_3448003521_rot90.jpg",
|
199 |
)
|
200 |
args = parser.parse_args()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
201 |
|
202 |
+
# get api version
|
203 |
+
get_api_version()
|
204 |
+
|
205 |
+
# request match
|
206 |
+
# for i in range(10):
|
207 |
+
# t1 = time.time()
|
208 |
+
# preds = send_request_match(args.image0, args.image1)
|
209 |
+
# t2 = time.time()
|
210 |
+
# print(
|
211 |
+
# "Time cost1: {} seconds, matched: {}".format(
|
212 |
+
# (t2 - t1), len(preds["mmkeypoints0_orig"])
|
213 |
+
# )
|
214 |
+
# )
|
215 |
+
|
216 |
+
# request extract
|
217 |
for i in range(10):
|
218 |
t1 = time.time()
|
219 |
+
preds = send_request_extract(args.image0)
|
220 |
t2 = time.time()
|
221 |
+
print(f"Time cost2: {(t2 - t1)} seconds")
|
222 |
|
223 |
+
# dump preds
|
224 |
with open("preds.pkl", "wb") as f:
|
225 |
pickle.dump(preds, f)
|
api/server.py
CHANGED
@@ -1,73 +1,435 @@
|
|
1 |
# server.py
|
|
|
|
|
2 |
import sys
|
|
|
3 |
from pathlib import Path
|
4 |
-
from typing import Union
|
5 |
|
|
|
|
|
6 |
import numpy as np
|
|
|
7 |
import uvicorn
|
8 |
from fastapi import FastAPI, File, UploadFile
|
|
|
9 |
from fastapi.responses import JSONResponse
|
10 |
from PIL import Image
|
11 |
|
12 |
-
sys.path.append(
|
13 |
-
from pydantic import BaseModel
|
14 |
|
15 |
-
from
|
16 |
-
from
|
|
|
|
|
|
|
|
|
17 |
|
|
|
18 |
|
19 |
-
|
20 |
-
|
21 |
-
|
22 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
23 |
|
24 |
|
25 |
class ImageMatchingService:
|
26 |
def __init__(self, conf: dict, device: str):
|
|
|
27 |
self.api = ImageMatchingAPI(conf=conf, device=device)
|
28 |
self.app = FastAPI()
|
29 |
self.register_routes()
|
30 |
|
31 |
def register_routes(self):
|
|
|
|
|
|
|
|
|
|
|
32 |
@self.app.post("/v1/match")
|
33 |
async def match(
|
34 |
image0: UploadFile = File(...), image1: UploadFile = File(...)
|
35 |
):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
36 |
try:
|
|
|
37 |
image0_array = self.load_image(image0)
|
38 |
image1_array = self.load_image(image1)
|
39 |
|
|
|
40 |
output = self.api(image0_array, image1_array)
|
41 |
|
|
|
42 |
skip_keys = ["image0_orig", "image1_orig"]
|
43 |
-
pred = self.filter_output(output, skip_keys)
|
44 |
|
|
|
|
|
|
|
|
|
45 |
return JSONResponse(content=pred)
|
46 |
except Exception as e:
|
|
|
47 |
return JSONResponse(content={"error": str(e)}, status_code=500)
|
48 |
|
49 |
@self.app.post("/v1/extract")
|
50 |
-
async def extract(
|
51 |
-
|
52 |
-
|
53 |
-
|
54 |
-
|
55 |
-
|
56 |
-
return JSONResponse(content=pred)
|
57 |
-
except Exception as e:
|
58 |
-
return JSONResponse(content={"error": str(e)}, status_code=500)
|
59 |
|
60 |
-
|
61 |
-
|
62 |
-
|
63 |
try:
|
64 |
-
|
65 |
-
|
66 |
-
|
67 |
-
|
68 |
-
|
69 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
70 |
except Exception as e:
|
|
|
71 |
return JSONResponse(content={"error": str(e)}, status_code=500)
|
72 |
|
73 |
def load_image(self, file_path: Union[str, UploadFile]) -> np.ndarray:
|
@@ -88,7 +450,9 @@ class ImageMatchingService:
|
|
88 |
image_array = np.array(img)
|
89 |
return image_array
|
90 |
|
91 |
-
def
|
|
|
|
|
92 |
pred = {}
|
93 |
for key, value in output.items():
|
94 |
if key in skip_keys:
|
|
|
1 |
# server.py
|
2 |
+
import base64
|
3 |
+
import io
|
4 |
import sys
|
5 |
+
import warnings
|
6 |
from pathlib import Path
|
7 |
+
from typing import Any, Dict, Optional, Union
|
8 |
|
9 |
+
import cv2
|
10 |
+
import matplotlib.pyplot as plt
|
11 |
import numpy as np
|
12 |
+
import torch
|
13 |
import uvicorn
|
14 |
from fastapi import FastAPI, File, UploadFile
|
15 |
+
from fastapi.exceptions import HTTPException
|
16 |
from fastapi.responses import JSONResponse
|
17 |
from PIL import Image
|
18 |
|
19 |
+
sys.path.append(str(Path(__file__).parents[1]))
|
|
|
20 |
|
21 |
+
from api.types import ImagesInput
|
22 |
+
from hloc import DEVICE, extract_features, logger, match_dense, match_features
|
23 |
+
from hloc.utils.viz import add_text, plot_keypoints
|
24 |
+
from ui import get_version
|
25 |
+
from ui.utils import filter_matches, get_feature_model, get_model
|
26 |
+
from ui.viz import display_matches, fig2im, plot_images
|
27 |
|
28 |
+
warnings.simplefilter("ignore")
|
29 |
|
30 |
+
|
31 |
+
def decode_base64_to_image(encoding):
|
32 |
+
if encoding.startswith("data:image/"):
|
33 |
+
encoding = encoding.split(";")[1].split(",")[1]
|
34 |
+
try:
|
35 |
+
image = Image.open(io.BytesIO(base64.b64decode(encoding)))
|
36 |
+
return image
|
37 |
+
except Exception as e:
|
38 |
+
logger.warning(f"API cannot decode image: {e}")
|
39 |
+
raise HTTPException(
|
40 |
+
status_code=500, detail="Invalid encoded image"
|
41 |
+
) from e
|
42 |
+
|
43 |
+
|
44 |
+
def to_base64_nparray(encoding: str) -> np.ndarray:
|
45 |
+
return np.array(decode_base64_to_image(encoding)).astype("uint8")
|
46 |
+
|
47 |
+
|
48 |
+
class ImageMatchingAPI(torch.nn.Module):
|
49 |
+
default_conf = {
|
50 |
+
"ransac": {
|
51 |
+
"enable": True,
|
52 |
+
"estimator": "poselib",
|
53 |
+
"geometry": "homography",
|
54 |
+
"method": "RANSAC",
|
55 |
+
"reproj_threshold": 3,
|
56 |
+
"confidence": 0.9999,
|
57 |
+
"max_iter": 10000,
|
58 |
+
},
|
59 |
+
}
|
60 |
+
|
61 |
+
def __init__(
|
62 |
+
self,
|
63 |
+
conf: dict = {},
|
64 |
+
device: str = "cpu",
|
65 |
+
detect_threshold: float = 0.015,
|
66 |
+
max_keypoints: int = 1024,
|
67 |
+
match_threshold: float = 0.2,
|
68 |
+
) -> None:
|
69 |
+
"""
|
70 |
+
Initializes an instance of the ImageMatchingAPI class.
|
71 |
+
|
72 |
+
Args:
|
73 |
+
conf (dict): A dictionary containing the configuration parameters.
|
74 |
+
device (str, optional): The device to use for computation. Defaults to "cpu".
|
75 |
+
detect_threshold (float, optional): The threshold for detecting keypoints. Defaults to 0.015.
|
76 |
+
max_keypoints (int, optional): The maximum number of keypoints to extract. Defaults to 1024.
|
77 |
+
match_threshold (float, optional): The threshold for matching keypoints. Defaults to 0.2.
|
78 |
+
|
79 |
+
Returns:
|
80 |
+
None
|
81 |
+
"""
|
82 |
+
super().__init__()
|
83 |
+
self.device = device
|
84 |
+
self.conf = {**self.default_conf, **conf}
|
85 |
+
self._updata_config(detect_threshold, max_keypoints, match_threshold)
|
86 |
+
self._init_models()
|
87 |
+
if device == "cuda":
|
88 |
+
memory_allocated = torch.cuda.memory_allocated(device)
|
89 |
+
memory_reserved = torch.cuda.memory_reserved(device)
|
90 |
+
logger.info(
|
91 |
+
f"GPU memory allocated: {memory_allocated / 1024**2:.3f} MB"
|
92 |
+
)
|
93 |
+
logger.info(
|
94 |
+
f"GPU memory reserved: {memory_reserved / 1024**2:.3f} MB"
|
95 |
+
)
|
96 |
+
self.pred = None
|
97 |
+
|
98 |
+
def parse_match_config(self, conf):
|
99 |
+
if conf["dense"]:
|
100 |
+
return {
|
101 |
+
**conf,
|
102 |
+
"matcher": match_dense.confs.get(
|
103 |
+
conf["matcher"]["model"]["name"]
|
104 |
+
),
|
105 |
+
"dense": True,
|
106 |
+
}
|
107 |
+
else:
|
108 |
+
return {
|
109 |
+
**conf,
|
110 |
+
"feature": extract_features.confs.get(
|
111 |
+
conf["feature"]["model"]["name"]
|
112 |
+
),
|
113 |
+
"matcher": match_features.confs.get(
|
114 |
+
conf["matcher"]["model"]["name"]
|
115 |
+
),
|
116 |
+
"dense": False,
|
117 |
+
}
|
118 |
+
|
119 |
+
def _updata_config(
|
120 |
+
self,
|
121 |
+
detect_threshold: float = 0.015,
|
122 |
+
max_keypoints: int = 1024,
|
123 |
+
match_threshold: float = 0.2,
|
124 |
+
):
|
125 |
+
self.dense = self.conf["dense"]
|
126 |
+
if self.conf["dense"]:
|
127 |
+
try:
|
128 |
+
self.conf["matcher"]["model"][
|
129 |
+
"match_threshold"
|
130 |
+
] = match_threshold
|
131 |
+
except TypeError as e:
|
132 |
+
logger.error(e)
|
133 |
+
else:
|
134 |
+
self.conf["feature"]["model"]["max_keypoints"] = max_keypoints
|
135 |
+
self.conf["feature"]["model"][
|
136 |
+
"keypoint_threshold"
|
137 |
+
] = detect_threshold
|
138 |
+
self.extract_conf = self.conf["feature"]
|
139 |
+
|
140 |
+
self.match_conf = self.conf["matcher"]
|
141 |
+
|
142 |
+
def _init_models(self):
|
143 |
+
# initialize matcher
|
144 |
+
self.matcher = get_model(self.match_conf)
|
145 |
+
# initialize extractor
|
146 |
+
if self.dense:
|
147 |
+
self.extractor = None
|
148 |
+
else:
|
149 |
+
self.extractor = get_feature_model(self.conf["feature"])
|
150 |
+
|
151 |
+
def _forward(self, img0, img1):
|
152 |
+
if self.dense:
|
153 |
+
pred = match_dense.match_images(
|
154 |
+
self.matcher,
|
155 |
+
img0,
|
156 |
+
img1,
|
157 |
+
self.match_conf["preprocessing"],
|
158 |
+
device=self.device,
|
159 |
+
)
|
160 |
+
last_fixed = "{}".format( # noqa: F841
|
161 |
+
self.match_conf["model"]["name"]
|
162 |
+
)
|
163 |
+
else:
|
164 |
+
pred0 = extract_features.extract(
|
165 |
+
self.extractor, img0, self.extract_conf["preprocessing"]
|
166 |
+
)
|
167 |
+
pred1 = extract_features.extract(
|
168 |
+
self.extractor, img1, self.extract_conf["preprocessing"]
|
169 |
+
)
|
170 |
+
pred = match_features.match_images(self.matcher, pred0, pred1)
|
171 |
+
return pred
|
172 |
+
|
173 |
+
@torch.inference_mode()
|
174 |
+
def extract(self, img0: np.ndarray, **kwargs) -> Dict[str, np.ndarray]:
|
175 |
+
"""Extract features from a single image.
|
176 |
+
|
177 |
+
Args:
|
178 |
+
img0 (np.ndarray): image
|
179 |
+
|
180 |
+
Returns:
|
181 |
+
Dict[str, np.ndarray]: feature dict
|
182 |
+
"""
|
183 |
+
|
184 |
+
# setting prams
|
185 |
+
self.extractor.conf["max_keypoints"] = kwargs.get("max_keypoints", 512)
|
186 |
+
self.extractor.conf["keypoint_threshold"] = kwargs.get(
|
187 |
+
"keypoint_threshold", 0.0
|
188 |
+
)
|
189 |
+
|
190 |
+
pred = extract_features.extract(
|
191 |
+
self.extractor, img0, self.extract_conf["preprocessing"]
|
192 |
+
)
|
193 |
+
pred = {
|
194 |
+
k: v.cpu().detach()[0].numpy() if isinstance(v, torch.Tensor) else v
|
195 |
+
for k, v in pred.items()
|
196 |
+
}
|
197 |
+
# back to origin scale
|
198 |
+
s0 = pred["original_size"] / pred["size"]
|
199 |
+
pred["keypoints_orig"] = (
|
200 |
+
match_features.scale_keypoints(pred["keypoints"] + 0.5, s0) - 0.5
|
201 |
+
)
|
202 |
+
# TODO: rotate back
|
203 |
+
|
204 |
+
binarize = kwargs.get("binarize", False)
|
205 |
+
if binarize:
|
206 |
+
assert "descriptors" in pred
|
207 |
+
pred["descriptors"] = (pred["descriptors"] > 0).astype(np.uint8)
|
208 |
+
pred["descriptors"] = pred["descriptors"].T # N x DIM
|
209 |
+
return pred
|
210 |
+
|
211 |
+
@torch.inference_mode()
|
212 |
+
def forward(
|
213 |
+
self,
|
214 |
+
img0: np.ndarray,
|
215 |
+
img1: np.ndarray,
|
216 |
+
) -> Dict[str, np.ndarray]:
|
217 |
+
"""
|
218 |
+
Forward pass of the image matching API.
|
219 |
+
|
220 |
+
Args:
|
221 |
+
img0: A 3D NumPy array of shape (H, W, C) representing the first image.
|
222 |
+
Values are in the range [0, 1] and are in RGB mode.
|
223 |
+
img1: A 3D NumPy array of shape (H, W, C) representing the second image.
|
224 |
+
Values are in the range [0, 1] and are in RGB mode.
|
225 |
+
|
226 |
+
Returns:
|
227 |
+
A dictionary containing the following keys:
|
228 |
+
- image0_orig: The original image 0.
|
229 |
+
- image1_orig: The original image 1.
|
230 |
+
- keypoints0_orig: The keypoints detected in image 0.
|
231 |
+
- keypoints1_orig: The keypoints detected in image 1.
|
232 |
+
- mkeypoints0_orig: The raw matches between image 0 and image 1.
|
233 |
+
- mkeypoints1_orig: The raw matches between image 1 and image 0.
|
234 |
+
- mmkeypoints0_orig: The RANSAC inliers in image 0.
|
235 |
+
- mmkeypoints1_orig: The RANSAC inliers in image 1.
|
236 |
+
- mconf: The confidence scores for the raw matches.
|
237 |
+
- mmconf: The confidence scores for the RANSAC inliers.
|
238 |
+
"""
|
239 |
+
# Take as input a pair of images (not a batch)
|
240 |
+
assert isinstance(img0, np.ndarray)
|
241 |
+
assert isinstance(img1, np.ndarray)
|
242 |
+
self.pred = self._forward(img0, img1)
|
243 |
+
if self.conf["ransac"]["enable"]:
|
244 |
+
self.pred = self._geometry_check(self.pred)
|
245 |
+
return self.pred
|
246 |
+
|
247 |
+
def _geometry_check(
|
248 |
+
self,
|
249 |
+
pred: Dict[str, Any],
|
250 |
+
) -> Dict[str, Any]:
|
251 |
+
"""
|
252 |
+
Filter matches using RANSAC. If keypoints are available, filter by keypoints.
|
253 |
+
If lines are available, filter by lines. If both keypoints and lines are
|
254 |
+
available, filter by keypoints.
|
255 |
+
|
256 |
+
Args:
|
257 |
+
pred (Dict[str, Any]): dict of matches, including original keypoints.
|
258 |
+
See :func:`filter_matches` for the expected keys.
|
259 |
+
|
260 |
+
Returns:
|
261 |
+
Dict[str, Any]: filtered matches
|
262 |
+
"""
|
263 |
+
pred = filter_matches(
|
264 |
+
pred,
|
265 |
+
ransac_method=self.conf["ransac"]["method"],
|
266 |
+
ransac_reproj_threshold=self.conf["ransac"]["reproj_threshold"],
|
267 |
+
ransac_confidence=self.conf["ransac"]["confidence"],
|
268 |
+
ransac_max_iter=self.conf["ransac"]["max_iter"],
|
269 |
+
)
|
270 |
+
return pred
|
271 |
+
|
272 |
+
def visualize(
|
273 |
+
self,
|
274 |
+
log_path: Optional[Path] = None,
|
275 |
+
) -> None:
|
276 |
+
"""
|
277 |
+
Visualize the matches.
|
278 |
+
|
279 |
+
Args:
|
280 |
+
log_path (Path, optional): The directory to save the images. Defaults to None.
|
281 |
+
|
282 |
+
Returns:
|
283 |
+
None
|
284 |
+
"""
|
285 |
+
if self.conf["dense"]:
|
286 |
+
postfix = str(self.conf["matcher"]["model"]["name"])
|
287 |
+
else:
|
288 |
+
postfix = "{}_{}".format(
|
289 |
+
str(self.conf["feature"]["model"]["name"]),
|
290 |
+
str(self.conf["matcher"]["model"]["name"]),
|
291 |
+
)
|
292 |
+
titles = [
|
293 |
+
"Image 0 - Keypoints",
|
294 |
+
"Image 1 - Keypoints",
|
295 |
+
]
|
296 |
+
pred: Dict[str, Any] = self.pred
|
297 |
+
image0: np.ndarray = pred["image0_orig"]
|
298 |
+
image1: np.ndarray = pred["image1_orig"]
|
299 |
+
output_keypoints: np.ndarray = plot_images(
|
300 |
+
[image0, image1], titles=titles, dpi=300
|
301 |
+
)
|
302 |
+
if (
|
303 |
+
"keypoints0_orig" in pred.keys()
|
304 |
+
and "keypoints1_orig" in pred.keys()
|
305 |
+
):
|
306 |
+
plot_keypoints([pred["keypoints0_orig"], pred["keypoints1_orig"]])
|
307 |
+
text: str = (
|
308 |
+
f"# keypoints0: {len(pred['keypoints0_orig'])} \n"
|
309 |
+
+ f"# keypoints1: {len(pred['keypoints1_orig'])}"
|
310 |
+
)
|
311 |
+
add_text(0, text, fs=15)
|
312 |
+
output_keypoints = fig2im(output_keypoints)
|
313 |
+
# plot images with raw matches
|
314 |
+
titles = [
|
315 |
+
"Image 0 - Raw matched keypoints",
|
316 |
+
"Image 1 - Raw matched keypoints",
|
317 |
+
]
|
318 |
+
output_matches_raw, num_matches_raw = display_matches(
|
319 |
+
pred, titles=titles, tag="KPTS_RAW"
|
320 |
+
)
|
321 |
+
# plot images with ransac matches
|
322 |
+
titles = [
|
323 |
+
"Image 0 - Ransac matched keypoints",
|
324 |
+
"Image 1 - Ransac matched keypoints",
|
325 |
+
]
|
326 |
+
output_matches_ransac, num_matches_ransac = display_matches(
|
327 |
+
pred, titles=titles, tag="KPTS_RANSAC"
|
328 |
+
)
|
329 |
+
if log_path is not None:
|
330 |
+
img_keypoints_path: Path = log_path / f"img_keypoints_{postfix}.png"
|
331 |
+
img_matches_raw_path: Path = (
|
332 |
+
log_path / f"img_matches_raw_{postfix}.png"
|
333 |
+
)
|
334 |
+
img_matches_ransac_path: Path = (
|
335 |
+
log_path / f"img_matches_ransac_{postfix}.png"
|
336 |
+
)
|
337 |
+
cv2.imwrite(
|
338 |
+
str(img_keypoints_path),
|
339 |
+
output_keypoints[:, :, ::-1].copy(), # RGB -> BGR
|
340 |
+
)
|
341 |
+
cv2.imwrite(
|
342 |
+
str(img_matches_raw_path),
|
343 |
+
output_matches_raw[:, :, ::-1].copy(), # RGB -> BGR
|
344 |
+
)
|
345 |
+
cv2.imwrite(
|
346 |
+
str(img_matches_ransac_path),
|
347 |
+
output_matches_ransac[:, :, ::-1].copy(), # RGB -> BGR
|
348 |
+
)
|
349 |
+
plt.close("all")
|
350 |
|
351 |
|
352 |
class ImageMatchingService:
|
353 |
def __init__(self, conf: dict, device: str):
|
354 |
+
self.conf = conf
|
355 |
self.api = ImageMatchingAPI(conf=conf, device=device)
|
356 |
self.app = FastAPI()
|
357 |
self.register_routes()
|
358 |
|
359 |
def register_routes(self):
|
360 |
+
|
361 |
+
@self.app.get("/version")
|
362 |
+
async def version():
|
363 |
+
return {"version": get_version()}
|
364 |
+
|
365 |
@self.app.post("/v1/match")
|
366 |
async def match(
|
367 |
image0: UploadFile = File(...), image1: UploadFile = File(...)
|
368 |
):
|
369 |
+
"""
|
370 |
+
Handle the image matching request and return the processed result.
|
371 |
+
|
372 |
+
Args:
|
373 |
+
image0 (UploadFile): The first image file for matching.
|
374 |
+
image1 (UploadFile): The second image file for matching.
|
375 |
+
|
376 |
+
Returns:
|
377 |
+
JSONResponse: A JSON response containing the filtered match results
|
378 |
+
or an error message in case of failure.
|
379 |
+
"""
|
380 |
try:
|
381 |
+
# Load the images from the uploaded files
|
382 |
image0_array = self.load_image(image0)
|
383 |
image1_array = self.load_image(image1)
|
384 |
|
385 |
+
# Perform image matching using the API
|
386 |
output = self.api(image0_array, image1_array)
|
387 |
|
388 |
+
# Keys to skip in the output
|
389 |
skip_keys = ["image0_orig", "image1_orig"]
|
|
|
390 |
|
391 |
+
# Postprocess the output to filter unwanted data
|
392 |
+
pred = self.postprocess(output, skip_keys)
|
393 |
+
|
394 |
+
# Return the filtered prediction as a JSON response
|
395 |
return JSONResponse(content=pred)
|
396 |
except Exception as e:
|
397 |
+
# Return an error message with status code 500 in case of exception
|
398 |
return JSONResponse(content={"error": str(e)}, status_code=500)
|
399 |
|
400 |
@self.app.post("/v1/extract")
|
401 |
+
async def extract(input_info: ImagesInput):
|
402 |
+
"""
|
403 |
+
Extract keypoints and descriptors from images.
|
404 |
+
|
405 |
+
Args:
|
406 |
+
input_info: An object containing the image data and options.
|
|
|
|
|
|
|
407 |
|
408 |
+
Returns:
|
409 |
+
A list of dictionaries containing the keypoints and descriptors.
|
410 |
+
"""
|
411 |
try:
|
412 |
+
preds = []
|
413 |
+
for i, input_image in enumerate(input_info.data):
|
414 |
+
# Load the image from the input data
|
415 |
+
image_array = to_base64_nparray(input_image)
|
416 |
+
# Extract keypoints and descriptors
|
417 |
+
output = self.api.extract(
|
418 |
+
image_array,
|
419 |
+
max_keypoints=input_info.max_keypoints[i],
|
420 |
+
binarize=input_info.binarize,
|
421 |
+
)
|
422 |
+
# Do not return the original image and image_orig
|
423 |
+
# skip_keys = ["image", "image_orig"]
|
424 |
+
skip_keys = []
|
425 |
+
|
426 |
+
# Postprocess the output
|
427 |
+
pred = self.postprocess(output, skip_keys)
|
428 |
+
preds.append(pred)
|
429 |
+
# Return the list of extracted features
|
430 |
+
return JSONResponse(content=preds)
|
431 |
except Exception as e:
|
432 |
+
# Return an error message if an exception occurs
|
433 |
return JSONResponse(content={"error": str(e)}, status_code=500)
|
434 |
|
435 |
def load_image(self, file_path: Union[str, UploadFile]) -> np.ndarray:
|
|
|
450 |
image_array = np.array(img)
|
451 |
return image_array
|
452 |
|
453 |
+
def postprocess(
|
454 |
+
self, output: dict, skip_keys: list, binarize: bool = True
|
455 |
+
) -> dict:
|
456 |
pred = {}
|
457 |
for key, value in output.items():
|
458 |
if key in skip_keys:
|
api/test/CMakeLists.txt
ADDED
@@ -0,0 +1,16 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
cmake_minimum_required(VERSION 3.10)
|
2 |
+
project(imatchui)
|
3 |
+
|
4 |
+
set(OpenCV_DIR /usr/include/opencv4)
|
5 |
+
find_package(OpenCV REQUIRED)
|
6 |
+
|
7 |
+
find_package(Boost REQUIRED COMPONENTS system)
|
8 |
+
if(Boost_FOUND)
|
9 |
+
include_directories(${Boost_INCLUDE_DIRS})
|
10 |
+
endif()
|
11 |
+
|
12 |
+
add_executable(client client.cpp)
|
13 |
+
|
14 |
+
target_include_directories(client PRIVATE ${Boost_LIBRARIES} ${OpenCV_INCLUDE_DIRS})
|
15 |
+
|
16 |
+
target_link_libraries(client PRIVATE curl jsoncpp b64 ${OpenCV_LIBS})
|
api/test/build_and_run.sh
ADDED
@@ -0,0 +1,16 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
# g++ main.cpp -I/usr/include/opencv4 -lcurl -ljsoncpp -lb64 -lopencv_core -lopencv_imgcodecs -o main
|
2 |
+
# sudo apt-get update
|
3 |
+
# sudo apt-get install libboost-all-dev -y
|
4 |
+
# sudo apt-get install libcurl4-openssl-dev libjsoncpp-dev libb64-dev libopencv-dev -y
|
5 |
+
|
6 |
+
cd build
|
7 |
+
cmake ..
|
8 |
+
make -j12
|
9 |
+
|
10 |
+
echo " ======== RUN DEMO ========"
|
11 |
+
|
12 |
+
./client
|
13 |
+
|
14 |
+
echo " ======== END DEMO ========"
|
15 |
+
|
16 |
+
cd ..
|
api/test/client.cpp
ADDED
@@ -0,0 +1,84 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
#include <curl/curl.h>
|
2 |
+
#include <opencv2/opencv.hpp>
|
3 |
+
#include "helper.h"
|
4 |
+
|
5 |
+
int main() {
|
6 |
+
std::string img_path = "../../../datasets/sacre_coeur/mapping_rot/02928139_3448003521_rot45.jpg";
|
7 |
+
cv::Mat original_img = cv::imread(img_path, cv::IMREAD_GRAYSCALE);
|
8 |
+
|
9 |
+
if (original_img.empty()) {
|
10 |
+
throw std::runtime_error("Failed to decode image");
|
11 |
+
}
|
12 |
+
|
13 |
+
// Convert the image to Base64
|
14 |
+
std::string base64_img = image_to_base64(original_img);
|
15 |
+
|
16 |
+
// Convert the Base64 back to an image
|
17 |
+
cv::Mat decoded_img = base64_to_image(base64_img);
|
18 |
+
cv::imwrite("decoded_image.jpg", decoded_img);
|
19 |
+
cv::imwrite("original_img.jpg", original_img);
|
20 |
+
|
21 |
+
// The images should be identical
|
22 |
+
if (cv::countNonZero(original_img != decoded_img) != 0) {
|
23 |
+
std::cerr << "The images are not identical" << std::endl;
|
24 |
+
return -1;
|
25 |
+
} else {
|
26 |
+
std::cout << "The images are identical!" << std::endl;
|
27 |
+
}
|
28 |
+
|
29 |
+
// construct params
|
30 |
+
APIParams params{
|
31 |
+
.data = {base64_img},
|
32 |
+
.max_keypoints = {100, 100},
|
33 |
+
.timestamps = {"0", "1"},
|
34 |
+
.grayscale = {0},
|
35 |
+
.image_hw = {{480, 640}, {240, 320}},
|
36 |
+
.feature_type = 0,
|
37 |
+
.rotates = {0.0f, 0.0f},
|
38 |
+
.scales = {1.0f, 1.0f},
|
39 |
+
.reference_points = {
|
40 |
+
{1.23e+2f, 1.2e+1f},
|
41 |
+
{5.0e-1f, 3.0e-1f},
|
42 |
+
{2.3e+2f, 2.2e+1f},
|
43 |
+
{6.0e-1f, 4.0e-1f}
|
44 |
+
},
|
45 |
+
.binarize = {1}
|
46 |
+
};
|
47 |
+
|
48 |
+
KeyPointResults kpts_results;
|
49 |
+
|
50 |
+
// Convert the parameters to JSON
|
51 |
+
Json::Value jsonData = paramsToJson(params);
|
52 |
+
std::string url = "http://127.0.0.1:8001/v1/extract";
|
53 |
+
Json::StreamWriterBuilder writer;
|
54 |
+
std::string output = Json::writeString(writer, jsonData);
|
55 |
+
|
56 |
+
CURL* curl;
|
57 |
+
CURLcode res;
|
58 |
+
std::string readBuffer;
|
59 |
+
|
60 |
+
curl_global_init(CURL_GLOBAL_DEFAULT);
|
61 |
+
curl = curl_easy_init();
|
62 |
+
if (curl) {
|
63 |
+
struct curl_slist* hs = NULL;
|
64 |
+
hs = curl_slist_append(hs, "Content-Type: application/json");
|
65 |
+
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, hs);
|
66 |
+
curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
|
67 |
+
curl_easy_setopt(curl, CURLOPT_POSTFIELDS, output.c_str());
|
68 |
+
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteCallback);
|
69 |
+
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &readBuffer);
|
70 |
+
res = curl_easy_perform(curl);
|
71 |
+
|
72 |
+
if (res != CURLE_OK)
|
73 |
+
fprintf(stderr, "curl_easy_perform() failed: %s\n",
|
74 |
+
curl_easy_strerror(res));
|
75 |
+
else {
|
76 |
+
// std::cout << "Response from server: " << readBuffer << std::endl;
|
77 |
+
kpts_results = decode_response(readBuffer);
|
78 |
+
}
|
79 |
+
curl_easy_cleanup(curl);
|
80 |
+
}
|
81 |
+
curl_global_cleanup();
|
82 |
+
|
83 |
+
return 0;
|
84 |
+
}
|
api/test/helper.h
ADDED
@@ -0,0 +1,410 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
|
2 |
+
#include <sstream>
|
3 |
+
#include <fstream>
|
4 |
+
#include <vector>
|
5 |
+
#include <b64/encode.h>
|
6 |
+
#include <jsoncpp/json/json.h>
|
7 |
+
#include <opencv2/opencv.hpp>
|
8 |
+
|
9 |
+
// base64 to image
|
10 |
+
#include <boost/archive/iterators/binary_from_base64.hpp>
|
11 |
+
#include <boost/archive/iterators/transform_width.hpp>
|
12 |
+
#include <boost/archive/iterators/base64_from_binary.hpp>
|
13 |
+
|
14 |
+
/// Parameters used in the API
|
15 |
+
struct APIParams {
|
16 |
+
/// A list of images, base64 encoded
|
17 |
+
std::vector<std::string> data;
|
18 |
+
|
19 |
+
/// The maximum number of keypoints to detect for each image
|
20 |
+
std::vector<int> max_keypoints;
|
21 |
+
|
22 |
+
/// The timestamps of the images
|
23 |
+
std::vector<std::string> timestamps;
|
24 |
+
|
25 |
+
/// Whether to convert the images to grayscale
|
26 |
+
bool grayscale;
|
27 |
+
|
28 |
+
/// The height and width of each image
|
29 |
+
std::vector<std::vector<int>> image_hw;
|
30 |
+
|
31 |
+
/// The type of feature detector to use
|
32 |
+
int feature_type;
|
33 |
+
|
34 |
+
/// The rotations of the images
|
35 |
+
std::vector<double> rotates;
|
36 |
+
|
37 |
+
/// The scales of the images
|
38 |
+
std::vector<double> scales;
|
39 |
+
|
40 |
+
/// The reference points of the images
|
41 |
+
std::vector<std::vector<float>> reference_points;
|
42 |
+
|
43 |
+
/// Whether to binarize the descriptors
|
44 |
+
bool binarize;
|
45 |
+
};
|
46 |
+
|
47 |
+
/**
|
48 |
+
* @brief Contains the results of a keypoint detector.
|
49 |
+
*
|
50 |
+
* @details Stores the keypoints and descriptors for each image.
|
51 |
+
*/
|
52 |
+
class KeyPointResults {
|
53 |
+
public:
|
54 |
+
KeyPointResults() {}
|
55 |
+
|
56 |
+
/**
|
57 |
+
* @brief Constructor.
|
58 |
+
*
|
59 |
+
* @param kp The keypoints for each image.
|
60 |
+
*/
|
61 |
+
KeyPointResults(const std::vector<std::vector<cv::KeyPoint>>& kp,
|
62 |
+
const std::vector<cv::Mat>& desc)
|
63 |
+
: keypoints(kp), descriptors(desc) {}
|
64 |
+
|
65 |
+
/**
|
66 |
+
* @brief Append keypoints to the result.
|
67 |
+
*
|
68 |
+
* @param kpts The keypoints to append.
|
69 |
+
*/
|
70 |
+
inline void append_keypoints(std::vector<cv::KeyPoint>& kpts) {
|
71 |
+
keypoints.emplace_back(kpts);
|
72 |
+
}
|
73 |
+
|
74 |
+
/**
|
75 |
+
* @brief Append descriptors to the result.
|
76 |
+
*
|
77 |
+
* @param desc The descriptors to append.
|
78 |
+
*/
|
79 |
+
inline void append_descriptors(cv::Mat& desc) {
|
80 |
+
descriptors.emplace_back(desc);
|
81 |
+
}
|
82 |
+
|
83 |
+
/**
|
84 |
+
* @brief Get the keypoints.
|
85 |
+
*
|
86 |
+
* @return The keypoints.
|
87 |
+
*/
|
88 |
+
inline std::vector<std::vector<cv::KeyPoint>> get_keypoints() {
|
89 |
+
return keypoints;
|
90 |
+
}
|
91 |
+
|
92 |
+
/**
|
93 |
+
* @brief Get the descriptors.
|
94 |
+
*
|
95 |
+
* @return The descriptors.
|
96 |
+
*/
|
97 |
+
inline std::vector<cv::Mat> get_descriptors() {
|
98 |
+
return descriptors;
|
99 |
+
}
|
100 |
+
|
101 |
+
private:
|
102 |
+
std::vector<std::vector<cv::KeyPoint>> keypoints;
|
103 |
+
std::vector<cv::Mat> descriptors;
|
104 |
+
std::vector<std::vector<float>> scores;
|
105 |
+
};
|
106 |
+
|
107 |
+
|
108 |
+
/**
|
109 |
+
* @brief Decodes a base64 encoded string.
|
110 |
+
*
|
111 |
+
* @param base64 The base64 encoded string to decode.
|
112 |
+
* @return The decoded string.
|
113 |
+
*/
|
114 |
+
std::string base64_decode(const std::string& base64) {
|
115 |
+
using namespace boost::archive::iterators;
|
116 |
+
using It = transform_width<binary_from_base64<std::string::const_iterator>, 8, 6>;
|
117 |
+
|
118 |
+
// Find the position of the last non-whitespace character
|
119 |
+
auto end = base64.find_last_not_of(" \t\n\r");
|
120 |
+
if (end != std::string::npos) {
|
121 |
+
// Move one past the last non-whitespace character
|
122 |
+
end += 1;
|
123 |
+
}
|
124 |
+
|
125 |
+
// Decode the base64 string and return the result
|
126 |
+
return std::string(It(base64.begin()), It(base64.begin() + end));
|
127 |
+
}
|
128 |
+
|
129 |
+
|
130 |
+
|
131 |
+
/**
|
132 |
+
* @brief Decodes a base64 string into an OpenCV image
|
133 |
+
*
|
134 |
+
* @param base64 The base64 encoded string
|
135 |
+
* @return The decoded OpenCV image
|
136 |
+
*/
|
137 |
+
cv::Mat base64_to_image(const std::string& base64) {
|
138 |
+
// Decode the base64 string
|
139 |
+
std::string decodedStr = base64_decode(base64);
|
140 |
+
|
141 |
+
// Decode the image
|
142 |
+
std::vector<uchar> data(decodedStr.begin(), decodedStr.end());
|
143 |
+
cv::Mat img = cv::imdecode(data, cv::IMREAD_GRAYSCALE);
|
144 |
+
|
145 |
+
// Check for errors
|
146 |
+
if (img.empty()) {
|
147 |
+
throw std::runtime_error("Failed to decode image");
|
148 |
+
}
|
149 |
+
|
150 |
+
return img;
|
151 |
+
}
|
152 |
+
|
153 |
+
|
154 |
+
/**
|
155 |
+
* @brief Encodes an OpenCV image into a base64 string
|
156 |
+
*
|
157 |
+
* This function takes an OpenCV image and encodes it into a base64 string.
|
158 |
+
* The image is first encoded as a PNG image, and then the resulting
|
159 |
+
* bytes are encoded as a base64 string.
|
160 |
+
*
|
161 |
+
* @param img The OpenCV image
|
162 |
+
* @return The base64 encoded string
|
163 |
+
*
|
164 |
+
* @throws std::runtime_error if the image is empty or encoding fails
|
165 |
+
*/
|
166 |
+
std::string image_to_base64(cv::Mat &img) {
|
167 |
+
if (img.empty()) {
|
168 |
+
throw std::runtime_error("Failed to read image");
|
169 |
+
}
|
170 |
+
|
171 |
+
// Encode the image as a PNG
|
172 |
+
std::vector<uchar> buf;
|
173 |
+
if (!cv::imencode(".png", img, buf)) {
|
174 |
+
throw std::runtime_error("Failed to encode image");
|
175 |
+
}
|
176 |
+
|
177 |
+
// Encode the bytes as a base64 string
|
178 |
+
using namespace boost::archive::iterators;
|
179 |
+
using It = base64_from_binary<transform_width<std::vector<uchar>::const_iterator, 6, 8>>;
|
180 |
+
std::string base64(It(buf.begin()), It(buf.end()));
|
181 |
+
|
182 |
+
// Pad the string with '=' characters to a multiple of 4 bytes
|
183 |
+
base64.append((3 - buf.size() % 3) % 3, '=');
|
184 |
+
|
185 |
+
return base64;
|
186 |
+
}
|
187 |
+
|
188 |
+
|
189 |
+
/**
|
190 |
+
* @brief Callback function for libcurl to write data to a string
|
191 |
+
*
|
192 |
+
* This function is used as a callback for libcurl to write data to a string.
|
193 |
+
* It takes the contents, size, and nmemb as parameters, and writes the data to
|
194 |
+
* the string.
|
195 |
+
*
|
196 |
+
* @param contents The data to write
|
197 |
+
* @param size The size of the data
|
198 |
+
* @param nmemb The number of members in the data
|
199 |
+
* @param s The string to write the data to
|
200 |
+
* @return The number of bytes written
|
201 |
+
*/
|
202 |
+
size_t WriteCallback(void* contents, size_t size, size_t nmemb, std::string* s) {
|
203 |
+
size_t newLength = size * nmemb;
|
204 |
+
try {
|
205 |
+
// Resize the string to fit the new data
|
206 |
+
s->resize(s->size() + newLength);
|
207 |
+
} catch (std::bad_alloc& e) {
|
208 |
+
// If there's an error allocating memory, return 0
|
209 |
+
return 0;
|
210 |
+
}
|
211 |
+
|
212 |
+
// Copy the data to the string
|
213 |
+
std::copy(static_cast<const char*>(contents),
|
214 |
+
static_cast<const char*>(contents) + newLength,
|
215 |
+
s->begin() + s->size() - newLength);
|
216 |
+
return newLength;
|
217 |
+
}
|
218 |
+
|
219 |
+
// Helper functions
|
220 |
+
|
221 |
+
/**
|
222 |
+
* @brief Helper function to convert a type to a Json::Value
|
223 |
+
*
|
224 |
+
* This function takes a value of type T and converts it to a Json::Value.
|
225 |
+
* It is used to simplify the process of converting a type to a Json::Value.
|
226 |
+
*
|
227 |
+
* @param val The value to convert
|
228 |
+
* @return The converted Json::Value
|
229 |
+
*/
|
230 |
+
template <typename T>
|
231 |
+
Json::Value toJson(const T& val) {
|
232 |
+
return Json::Value(val);
|
233 |
+
}
|
234 |
+
|
235 |
+
/**
|
236 |
+
* @brief Converts a vector to a Json::Value
|
237 |
+
*
|
238 |
+
* This function takes a vector of type T and converts it to a Json::Value.
|
239 |
+
* Each element in the vector is appended to the Json::Value array.
|
240 |
+
*
|
241 |
+
* @param vec The vector to convert to Json::Value
|
242 |
+
* @return The Json::Value representing the vector
|
243 |
+
*/
|
244 |
+
template <typename T>
|
245 |
+
Json::Value vectorToJson(const std::vector<T>& vec) {
|
246 |
+
Json::Value json(Json::arrayValue);
|
247 |
+
for (const auto& item : vec) {
|
248 |
+
json.append(item);
|
249 |
+
}
|
250 |
+
return json;
|
251 |
+
}
|
252 |
+
|
253 |
+
/**
|
254 |
+
* @brief Converts a nested vector to a Json::Value
|
255 |
+
*
|
256 |
+
* This function takes a nested vector of type T and converts it to a Json::Value.
|
257 |
+
* Each sub-vector is converted to a Json::Value array and appended to the main Json::Value array.
|
258 |
+
*
|
259 |
+
* @param vec The nested vector to convert to Json::Value
|
260 |
+
* @return The Json::Value representing the nested vector
|
261 |
+
*/
|
262 |
+
template <typename T>
|
263 |
+
Json::Value nestedVectorToJson(const std::vector<std::vector<T>>& vec) {
|
264 |
+
Json::Value json(Json::arrayValue);
|
265 |
+
for (const auto& subVec : vec) {
|
266 |
+
json.append(vectorToJson(subVec));
|
267 |
+
}
|
268 |
+
return json;
|
269 |
+
}
|
270 |
+
|
271 |
+
|
272 |
+
|
273 |
+
/**
|
274 |
+
* @brief Converts the APIParams struct to a Json::Value
|
275 |
+
*
|
276 |
+
* This function takes an APIParams struct and converts it to a Json::Value.
|
277 |
+
* The Json::Value is a JSON object with the following fields:
|
278 |
+
* - data: a JSON array of base64 encoded images
|
279 |
+
* - max_keypoints: a JSON array of integers, max number of keypoints for each image
|
280 |
+
* - timestamps: a JSON array of timestamps, one for each image
|
281 |
+
* - grayscale: a JSON boolean, whether to convert images to grayscale
|
282 |
+
* - image_hw: a nested JSON array, each sub-array contains the height and width of an image
|
283 |
+
* - feature_type: a JSON integer, the type of feature detector to use
|
284 |
+
* - rotates: a JSON array of doubles, the rotation of each image
|
285 |
+
* - scales: a JSON array of doubles, the scale of each image
|
286 |
+
* - reference_points: a nested JSON array, each sub-array contains the reference points of an image
|
287 |
+
* - binarize: a JSON boolean, whether to binarize the descriptors
|
288 |
+
*
|
289 |
+
* @param params The APIParams struct to convert
|
290 |
+
* @return The Json::Value representing the APIParams struct
|
291 |
+
*/
|
292 |
+
Json::Value paramsToJson(const APIParams& params) {
|
293 |
+
Json::Value json;
|
294 |
+
json["data"] = vectorToJson(params.data);
|
295 |
+
json["max_keypoints"] = vectorToJson(params.max_keypoints);
|
296 |
+
json["timestamps"] = vectorToJson(params.timestamps);
|
297 |
+
json["grayscale"] = toJson(params.grayscale);
|
298 |
+
json["image_hw"] = nestedVectorToJson(params.image_hw);
|
299 |
+
json["feature_type"] = toJson(params.feature_type);
|
300 |
+
json["rotates"] = vectorToJson(params.rotates);
|
301 |
+
json["scales"] = vectorToJson(params.scales);
|
302 |
+
json["reference_points"] = nestedVectorToJson(params.reference_points);
|
303 |
+
json["binarize"] = toJson(params.binarize);
|
304 |
+
return json;
|
305 |
+
}
|
306 |
+
|
307 |
+
template<typename T>
|
308 |
+
cv::Mat jsonToMat(Json::Value json) {
|
309 |
+
int rows = json.size();
|
310 |
+
int cols = json[0].size();
|
311 |
+
|
312 |
+
// Create a single array to hold all the data.
|
313 |
+
std::vector<T> data;
|
314 |
+
data.reserve(rows * cols);
|
315 |
+
|
316 |
+
for (int i = 0; i < rows; i++) {
|
317 |
+
for (int j = 0; j < cols; j++) {
|
318 |
+
data.push_back(static_cast<T>(json[i][j].asInt()));
|
319 |
+
}
|
320 |
+
}
|
321 |
+
|
322 |
+
// Create a cv::Mat object that points to the data.
|
323 |
+
cv::Mat mat(rows, cols, CV_8UC1, data.data()); // Change the type if necessary.
|
324 |
+
// cv::Mat mat(cols, rows,CV_8UC1, data.data()); // Change the type if necessary.
|
325 |
+
|
326 |
+
return mat;
|
327 |
+
}
|
328 |
+
|
329 |
+
|
330 |
+
|
331 |
+
/**
|
332 |
+
* @brief Decodes the response of the server and prints the keypoints
|
333 |
+
*
|
334 |
+
* This function takes the response of the server, a JSON string, and decodes
|
335 |
+
* it. It then prints the keypoints and draws them on the original image.
|
336 |
+
*
|
337 |
+
* @param response The response of the server
|
338 |
+
* @return The keypoints and descriptors
|
339 |
+
*/
|
340 |
+
KeyPointResults decode_response(const std::string& response, bool viz=true) {
|
341 |
+
Json::CharReaderBuilder builder;
|
342 |
+
Json::CharReader* reader = builder.newCharReader();
|
343 |
+
|
344 |
+
Json::Value jsonData;
|
345 |
+
std::string errors;
|
346 |
+
|
347 |
+
// Parse the JSON response
|
348 |
+
bool parsingSuccessful = reader->parse(response.c_str(),
|
349 |
+
response.c_str() + response.size(), &jsonData, &errors);
|
350 |
+
delete reader;
|
351 |
+
|
352 |
+
if (!parsingSuccessful) {
|
353 |
+
// Handle error
|
354 |
+
std::cout << "Failed to parse the JSON, errors:" << std::endl;
|
355 |
+
std::cout << errors << std::endl;
|
356 |
+
return KeyPointResults();
|
357 |
+
}
|
358 |
+
|
359 |
+
KeyPointResults kpts_results;
|
360 |
+
|
361 |
+
// Iterate over the images
|
362 |
+
for (const auto& jsonItem : jsonData) {
|
363 |
+
auto jkeypoints = jsonItem["keypoints"];
|
364 |
+
auto jkeypoints_orig = jsonItem["keypoints_orig"];
|
365 |
+
auto jdescriptors = jsonItem["descriptors"];
|
366 |
+
auto jscores = jsonItem["scores"];
|
367 |
+
auto jimageSize = jsonItem["image_size"];
|
368 |
+
auto joriginalSize = jsonItem["original_size"];
|
369 |
+
auto jsize = jsonItem["size"];
|
370 |
+
|
371 |
+
std::vector<cv::KeyPoint> vkeypoints;
|
372 |
+
std::vector<float> vscores;
|
373 |
+
|
374 |
+
// Iterate over the keypoints
|
375 |
+
int counter = 0;
|
376 |
+
for (const auto& keypoint : jkeypoints_orig) {
|
377 |
+
if (counter < 10) {
|
378 |
+
// Print the first 10 keypoints
|
379 |
+
std::cout << keypoint[0].asFloat() << ", "
|
380 |
+
<< keypoint[1].asFloat() << std::endl;
|
381 |
+
}
|
382 |
+
counter++;
|
383 |
+
// Convert the Json::Value to a cv::KeyPoint
|
384 |
+
vkeypoints.emplace_back(cv::KeyPoint(keypoint[0].asFloat(),
|
385 |
+
keypoint[1].asFloat(), 0.0));
|
386 |
+
}
|
387 |
+
|
388 |
+
if (viz && jsonItem.isMember("image_orig")) {
|
389 |
+
|
390 |
+
auto jimg_orig = jsonItem["image_orig"];
|
391 |
+
cv::Mat img = jsonToMat<uchar>(jimg_orig);
|
392 |
+
cv::imwrite("viz_image_orig.jpg", img);
|
393 |
+
|
394 |
+
// Draw keypoints on the image
|
395 |
+
cv::Mat imgWithKeypoints;
|
396 |
+
cv::drawKeypoints(img, vkeypoints,
|
397 |
+
imgWithKeypoints, cv::Scalar(0, 0, 255));
|
398 |
+
|
399 |
+
// Write the image with keypoints
|
400 |
+
std::string filename = "viz_image_orig_keypoints.jpg";
|
401 |
+
cv::imwrite(filename, imgWithKeypoints);
|
402 |
+
}
|
403 |
+
|
404 |
+
// Iterate over the descriptors
|
405 |
+
cv::Mat descriptors = jsonToMat<uchar>(jdescriptors);
|
406 |
+
kpts_results.append_keypoints(vkeypoints);
|
407 |
+
kpts_results.append_descriptors(descriptors);
|
408 |
+
}
|
409 |
+
return kpts_results;
|
410 |
+
}
|
api/types.py
ADDED
@@ -0,0 +1,16 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
from typing import List
|
2 |
+
|
3 |
+
from pydantic import BaseModel
|
4 |
+
|
5 |
+
|
6 |
+
class ImagesInput(BaseModel):
|
7 |
+
data: List[str] = []
|
8 |
+
max_keypoints: List[int] = []
|
9 |
+
timestamps: List[str] = []
|
10 |
+
grayscale: bool = False
|
11 |
+
image_hw: List[List[int]] = [[], []]
|
12 |
+
feature_type: int = 0
|
13 |
+
rotates: List[float] = []
|
14 |
+
scales: List[float] = []
|
15 |
+
reference_points: List[List[float]] = []
|
16 |
+
binarize: bool = False
|
requirements.txt
CHANGED
@@ -2,8 +2,7 @@ e2cnn
|
|
2 |
einops
|
3 |
easydict
|
4 |
gdown
|
5 |
-
gradio==
|
6 |
-
gradio_client==1.3.0
|
7 |
h5py
|
8 |
huggingface_hub
|
9 |
imageio
|
|
|
2 |
einops
|
3 |
easydict
|
4 |
gdown
|
5 |
+
gradio==5.1.0
|
|
|
6 |
h5py
|
7 |
huggingface_hub
|
8 |
imageio
|
test_app_cli.py
CHANGED
@@ -1,12 +1,13 @@
|
|
|
|
|
|
|
|
1 |
import cv2
|
|
|
2 |
from hloc import logger
|
3 |
-
from ui.utils import
|
4 |
-
|
5 |
-
|
6 |
-
|
7 |
-
ROOT,
|
8 |
-
)
|
9 |
-
from ui.api import ImageMatchingAPI
|
10 |
|
11 |
|
12 |
def test_all(config: dict = None):
|
@@ -68,7 +69,7 @@ def test_one():
|
|
68 |
"dense": False,
|
69 |
}
|
70 |
api = ImageMatchingAPI(conf=conf, device=DEVICE)
|
71 |
-
|
72 |
log_path = ROOT / "experiments" / "one"
|
73 |
log_path.mkdir(exist_ok=True, parents=True)
|
74 |
api.visualize(log_path=log_path)
|
|
|
1 |
+
import sys
|
2 |
+
from pathlib import Path
|
3 |
+
|
4 |
import cv2
|
5 |
+
|
6 |
from hloc import logger
|
7 |
+
from ui.utils import DEVICE, ROOT, get_matcher_zoo, load_config
|
8 |
+
|
9 |
+
sys.path.append(str(Path(__file__).parents[1]))
|
10 |
+
from api.server import ImageMatchingAPI
|
|
|
|
|
|
|
11 |
|
12 |
|
13 |
def test_all(config: dict = None):
|
|
|
69 |
"dense": False,
|
70 |
}
|
71 |
api = ImageMatchingAPI(conf=conf, device=DEVICE)
|
72 |
+
api(image0, image1)
|
73 |
log_path = ROOT / "experiments" / "one"
|
74 |
log_path.mkdir(exist_ok=True, parents=True)
|
75 |
api.visualize(log_path=log_path)
|
ui/__init__.py
CHANGED
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
__version__ = "1.0.1"
|
2 |
+
|
3 |
+
|
4 |
+
def get_version():
|
5 |
+
return __version__
|
ui/api.py
DELETED
@@ -1,293 +0,0 @@
|
|
1 |
-
import warnings
|
2 |
-
from pathlib import Path
|
3 |
-
from typing import Any, Dict, Optional
|
4 |
-
|
5 |
-
import cv2
|
6 |
-
import matplotlib.pyplot as plt
|
7 |
-
import numpy as np
|
8 |
-
import torch
|
9 |
-
|
10 |
-
from hloc import extract_features, logger, match_dense, match_features
|
11 |
-
from hloc.utils.viz import add_text, plot_keypoints
|
12 |
-
|
13 |
-
from .utils import (
|
14 |
-
ROOT,
|
15 |
-
filter_matches,
|
16 |
-
get_feature_model,
|
17 |
-
get_model,
|
18 |
-
load_config,
|
19 |
-
)
|
20 |
-
from .viz import display_matches, fig2im, plot_images
|
21 |
-
|
22 |
-
warnings.simplefilter("ignore")
|
23 |
-
|
24 |
-
|
25 |
-
class ImageMatchingAPI(torch.nn.Module):
|
26 |
-
default_conf = {
|
27 |
-
"ransac": {
|
28 |
-
"enable": True,
|
29 |
-
"estimator": "poselib",
|
30 |
-
"geometry": "homography",
|
31 |
-
"method": "RANSAC",
|
32 |
-
"reproj_threshold": 3,
|
33 |
-
"confidence": 0.9999,
|
34 |
-
"max_iter": 10000,
|
35 |
-
},
|
36 |
-
}
|
37 |
-
|
38 |
-
def __init__(
|
39 |
-
self,
|
40 |
-
conf: dict = {},
|
41 |
-
device: str = "cpu",
|
42 |
-
detect_threshold: float = 0.015,
|
43 |
-
max_keypoints: int = 1024,
|
44 |
-
match_threshold: float = 0.2,
|
45 |
-
) -> None:
|
46 |
-
"""
|
47 |
-
Initializes an instance of the ImageMatchingAPI class.
|
48 |
-
|
49 |
-
Args:
|
50 |
-
conf (dict): A dictionary containing the configuration parameters.
|
51 |
-
device (str, optional): The device to use for computation. Defaults to "cpu".
|
52 |
-
detect_threshold (float, optional): The threshold for detecting keypoints. Defaults to 0.015.
|
53 |
-
max_keypoints (int, optional): The maximum number of keypoints to extract. Defaults to 1024.
|
54 |
-
match_threshold (float, optional): The threshold for matching keypoints. Defaults to 0.2.
|
55 |
-
|
56 |
-
Returns:
|
57 |
-
None
|
58 |
-
"""
|
59 |
-
super().__init__()
|
60 |
-
self.device = device
|
61 |
-
self.conf = {**self.default_conf, **conf}
|
62 |
-
self._updata_config(detect_threshold, max_keypoints, match_threshold)
|
63 |
-
self._init_models()
|
64 |
-
if device == "cuda":
|
65 |
-
memory_allocated = torch.cuda.memory_allocated(device)
|
66 |
-
memory_reserved = torch.cuda.memory_reserved(device)
|
67 |
-
logger.info(
|
68 |
-
f"GPU memory allocated: {memory_allocated / 1024**2:.3f} MB"
|
69 |
-
)
|
70 |
-
logger.info(
|
71 |
-
f"GPU memory reserved: {memory_reserved / 1024**2:.3f} MB"
|
72 |
-
)
|
73 |
-
self.pred = None
|
74 |
-
|
75 |
-
def parse_match_config(self, conf):
|
76 |
-
if conf["dense"]:
|
77 |
-
return {
|
78 |
-
**conf,
|
79 |
-
"matcher": match_dense.confs.get(
|
80 |
-
conf["matcher"]["model"]["name"]
|
81 |
-
),
|
82 |
-
"dense": True,
|
83 |
-
}
|
84 |
-
else:
|
85 |
-
return {
|
86 |
-
**conf,
|
87 |
-
"feature": extract_features.confs.get(
|
88 |
-
conf["feature"]["model"]["name"]
|
89 |
-
),
|
90 |
-
"matcher": match_features.confs.get(
|
91 |
-
conf["matcher"]["model"]["name"]
|
92 |
-
),
|
93 |
-
"dense": False,
|
94 |
-
}
|
95 |
-
|
96 |
-
def _updata_config(
|
97 |
-
self,
|
98 |
-
detect_threshold: float = 0.015,
|
99 |
-
max_keypoints: int = 1024,
|
100 |
-
match_threshold: float = 0.2,
|
101 |
-
):
|
102 |
-
self.dense = self.conf["dense"]
|
103 |
-
if self.conf["dense"]:
|
104 |
-
try:
|
105 |
-
self.conf["matcher"]["model"][
|
106 |
-
"match_threshold"
|
107 |
-
] = match_threshold
|
108 |
-
except TypeError as e:
|
109 |
-
logger.error(e)
|
110 |
-
else:
|
111 |
-
self.conf["feature"]["model"]["max_keypoints"] = max_keypoints
|
112 |
-
self.conf["feature"]["model"][
|
113 |
-
"keypoint_threshold"
|
114 |
-
] = detect_threshold
|
115 |
-
self.extract_conf = self.conf["feature"]
|
116 |
-
|
117 |
-
self.match_conf = self.conf["matcher"]
|
118 |
-
|
119 |
-
def _init_models(self):
|
120 |
-
# initialize matcher
|
121 |
-
self.matcher = get_model(self.match_conf)
|
122 |
-
# initialize extractor
|
123 |
-
if self.dense:
|
124 |
-
self.extractor = None
|
125 |
-
else:
|
126 |
-
self.extractor = get_feature_model(self.conf["feature"])
|
127 |
-
|
128 |
-
def _forward(self, img0, img1):
|
129 |
-
if self.dense:
|
130 |
-
pred = match_dense.match_images(
|
131 |
-
self.matcher,
|
132 |
-
img0,
|
133 |
-
img1,
|
134 |
-
self.match_conf["preprocessing"],
|
135 |
-
device=self.device,
|
136 |
-
)
|
137 |
-
last_fixed = "{}".format( # noqa: F841
|
138 |
-
self.match_conf["model"]["name"]
|
139 |
-
)
|
140 |
-
else:
|
141 |
-
pred0 = extract_features.extract(
|
142 |
-
self.extractor, img0, self.extract_conf["preprocessing"]
|
143 |
-
)
|
144 |
-
pred1 = extract_features.extract(
|
145 |
-
self.extractor, img1, self.extract_conf["preprocessing"]
|
146 |
-
)
|
147 |
-
pred = match_features.match_images(self.matcher, pred0, pred1)
|
148 |
-
return pred
|
149 |
-
|
150 |
-
@torch.inference_mode()
|
151 |
-
def forward(
|
152 |
-
self,
|
153 |
-
img0: np.ndarray,
|
154 |
-
img1: np.ndarray,
|
155 |
-
) -> Dict[str, np.ndarray]:
|
156 |
-
"""
|
157 |
-
Forward pass of the image matching API.
|
158 |
-
|
159 |
-
Args:
|
160 |
-
img0: A 3D NumPy array of shape (H, W, C) representing the first image.
|
161 |
-
Values are in the range [0, 1] and are in RGB mode.
|
162 |
-
img1: A 3D NumPy array of shape (H, W, C) representing the second image.
|
163 |
-
Values are in the range [0, 1] and are in RGB mode.
|
164 |
-
|
165 |
-
Returns:
|
166 |
-
A dictionary containing the following keys:
|
167 |
-
- image0_orig: The original image 0.
|
168 |
-
- image1_orig: The original image 1.
|
169 |
-
- keypoints0_orig: The keypoints detected in image 0.
|
170 |
-
- keypoints1_orig: The keypoints detected in image 1.
|
171 |
-
- mkeypoints0_orig: The raw matches between image 0 and image 1.
|
172 |
-
- mkeypoints1_orig: The raw matches between image 1 and image 0.
|
173 |
-
- mmkeypoints0_orig: The RANSAC inliers in image 0.
|
174 |
-
- mmkeypoints1_orig: The RANSAC inliers in image 1.
|
175 |
-
- mconf: The confidence scores for the raw matches.
|
176 |
-
- mmconf: The confidence scores for the RANSAC inliers.
|
177 |
-
"""
|
178 |
-
# Take as input a pair of images (not a batch)
|
179 |
-
assert isinstance(img0, np.ndarray)
|
180 |
-
assert isinstance(img1, np.ndarray)
|
181 |
-
self.pred = self._forward(img0, img1)
|
182 |
-
if self.conf["ransac"]["enable"]:
|
183 |
-
self.pred = self._geometry_check(self.pred)
|
184 |
-
return self.pred
|
185 |
-
|
186 |
-
def _geometry_check(
|
187 |
-
self,
|
188 |
-
pred: Dict[str, Any],
|
189 |
-
) -> Dict[str, Any]:
|
190 |
-
"""
|
191 |
-
Filter matches using RANSAC. If keypoints are available, filter by keypoints.
|
192 |
-
If lines are available, filter by lines. If both keypoints and lines are
|
193 |
-
available, filter by keypoints.
|
194 |
-
|
195 |
-
Args:
|
196 |
-
pred (Dict[str, Any]): dict of matches, including original keypoints.
|
197 |
-
See :func:`filter_matches` for the expected keys.
|
198 |
-
|
199 |
-
Returns:
|
200 |
-
Dict[str, Any]: filtered matches
|
201 |
-
"""
|
202 |
-
pred = filter_matches(
|
203 |
-
pred,
|
204 |
-
ransac_method=self.conf["ransac"]["method"],
|
205 |
-
ransac_reproj_threshold=self.conf["ransac"]["reproj_threshold"],
|
206 |
-
ransac_confidence=self.conf["ransac"]["confidence"],
|
207 |
-
ransac_max_iter=self.conf["ransac"]["max_iter"],
|
208 |
-
)
|
209 |
-
return pred
|
210 |
-
|
211 |
-
def visualize(
|
212 |
-
self,
|
213 |
-
log_path: Optional[Path] = None,
|
214 |
-
) -> None:
|
215 |
-
"""
|
216 |
-
Visualize the matches.
|
217 |
-
|
218 |
-
Args:
|
219 |
-
log_path (Path, optional): The directory to save the images. Defaults to None.
|
220 |
-
|
221 |
-
Returns:
|
222 |
-
None
|
223 |
-
"""
|
224 |
-
if self.conf["dense"]:
|
225 |
-
postfix = str(self.conf["matcher"]["model"]["name"])
|
226 |
-
else:
|
227 |
-
postfix = "{}_{}".format(
|
228 |
-
str(self.conf["feature"]["model"]["name"]),
|
229 |
-
str(self.conf["matcher"]["model"]["name"]),
|
230 |
-
)
|
231 |
-
titles = [
|
232 |
-
"Image 0 - Keypoints",
|
233 |
-
"Image 1 - Keypoints",
|
234 |
-
]
|
235 |
-
pred: Dict[str, Any] = self.pred
|
236 |
-
image0: np.ndarray = pred["image0_orig"]
|
237 |
-
image1: np.ndarray = pred["image1_orig"]
|
238 |
-
output_keypoints: np.ndarray = plot_images(
|
239 |
-
[image0, image1], titles=titles, dpi=300
|
240 |
-
)
|
241 |
-
if (
|
242 |
-
"keypoints0_orig" in pred.keys()
|
243 |
-
and "keypoints1_orig" in pred.keys()
|
244 |
-
):
|
245 |
-
plot_keypoints([pred["keypoints0_orig"], pred["keypoints1_orig"]])
|
246 |
-
text: str = (
|
247 |
-
f"# keypoints0: {len(pred['keypoints0_orig'])} \n"
|
248 |
-
+ f"# keypoints1: {len(pred['keypoints1_orig'])}"
|
249 |
-
)
|
250 |
-
add_text(0, text, fs=15)
|
251 |
-
output_keypoints = fig2im(output_keypoints)
|
252 |
-
# plot images with raw matches
|
253 |
-
titles = [
|
254 |
-
"Image 0 - Raw matched keypoints",
|
255 |
-
"Image 1 - Raw matched keypoints",
|
256 |
-
]
|
257 |
-
output_matches_raw, num_matches_raw = display_matches(
|
258 |
-
pred, titles=titles, tag="KPTS_RAW"
|
259 |
-
)
|
260 |
-
# plot images with ransac matches
|
261 |
-
titles = [
|
262 |
-
"Image 0 - Ransac matched keypoints",
|
263 |
-
"Image 1 - Ransac matched keypoints",
|
264 |
-
]
|
265 |
-
output_matches_ransac, num_matches_ransac = display_matches(
|
266 |
-
pred, titles=titles, tag="KPTS_RANSAC"
|
267 |
-
)
|
268 |
-
if log_path is not None:
|
269 |
-
img_keypoints_path: Path = log_path / f"img_keypoints_{postfix}.png"
|
270 |
-
img_matches_raw_path: Path = (
|
271 |
-
log_path / f"img_matches_raw_{postfix}.png"
|
272 |
-
)
|
273 |
-
img_matches_ransac_path: Path = (
|
274 |
-
log_path / f"img_matches_ransac_{postfix}.png"
|
275 |
-
)
|
276 |
-
cv2.imwrite(
|
277 |
-
str(img_keypoints_path),
|
278 |
-
output_keypoints[:, :, ::-1].copy(), # RGB -> BGR
|
279 |
-
)
|
280 |
-
cv2.imwrite(
|
281 |
-
str(img_matches_raw_path),
|
282 |
-
output_matches_raw[:, :, ::-1].copy(), # RGB -> BGR
|
283 |
-
)
|
284 |
-
cv2.imwrite(
|
285 |
-
str(img_matches_ransac_path),
|
286 |
-
output_matches_ransac[:, :, ::-1].copy(), # RGB -> BGR
|
287 |
-
)
|
288 |
-
plt.close("all")
|
289 |
-
|
290 |
-
|
291 |
-
if __name__ == "__main__":
|
292 |
-
config = load_config(ROOT / "ui/config.yaml")
|
293 |
-
api = ImageMatchingAPI(config)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
ui/app_class.py
CHANGED
@@ -1,3 +1,4 @@
|
|
|
|
1 |
from pathlib import Path
|
2 |
from typing import Any, Dict, Optional, Tuple
|
3 |
|
@@ -6,7 +7,8 @@ import numpy as np
|
|
6 |
from easydict import EasyDict as edict
|
7 |
from omegaconf import OmegaConf
|
8 |
|
9 |
-
|
|
|
10 |
from ui.sfm import SfmEngine
|
11 |
from ui.utils import (
|
12 |
GRADIO_VERSION,
|
@@ -272,24 +274,6 @@ class ImageMatchingApp:
|
|
272 |
self.display_supported_algorithms()
|
273 |
|
274 |
with gr.Column():
|
275 |
-
with gr.Accordion("Open for More: Logs", open=False):
|
276 |
-
logs = gr.Textbox(
|
277 |
-
placeholder="\n" * 10,
|
278 |
-
label="Logs",
|
279 |
-
info="Verbose from inference will be displayed below.",
|
280 |
-
lines=10,
|
281 |
-
max_lines=10,
|
282 |
-
autoscroll=True,
|
283 |
-
elem_id="logs",
|
284 |
-
show_copy_button=True,
|
285 |
-
container=True,
|
286 |
-
elem_classes="logs_class",
|
287 |
-
)
|
288 |
-
self.app.load(read_logs, None, logs, every=1)
|
289 |
-
btn_clear_logs = gr.Button(
|
290 |
-
"Clear logs", elem_id="logs-button"
|
291 |
-
)
|
292 |
-
btn_clear_logs.click(flush_logs, [], [])
|
293 |
|
294 |
with gr.Accordion(
|
295 |
"Open for More: Keypoints", open=True
|
@@ -523,7 +507,7 @@ class ImageMatchingApp:
|
|
523 |
key: str = list(self.matcher_zoo.keys())[
|
524 |
0
|
525 |
] # Get the first key from matcher_zoo
|
526 |
-
flush_logs()
|
527 |
return (
|
528 |
None, # image0: Optional[np.ndarray]
|
529 |
None, # image1: Optional[np.ndarray]
|
|
|
1 |
+
import sys
|
2 |
from pathlib import Path
|
3 |
from typing import Any, Dict, Optional, Tuple
|
4 |
|
|
|
7 |
from easydict import EasyDict as edict
|
8 |
from omegaconf import OmegaConf
|
9 |
|
10 |
+
sys.path.append(str(Path(__file__).parents[1]))
|
11 |
+
|
12 |
from ui.sfm import SfmEngine
|
13 |
from ui.utils import (
|
14 |
GRADIO_VERSION,
|
|
|
274 |
self.display_supported_algorithms()
|
275 |
|
276 |
with gr.Column():
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
277 |
|
278 |
with gr.Accordion(
|
279 |
"Open for More: Keypoints", open=True
|
|
|
507 |
key: str = list(self.matcher_zoo.keys())[
|
508 |
0
|
509 |
] # Get the first key from matcher_zoo
|
510 |
+
# flush_logs()
|
511 |
return (
|
512 |
None, # image0: Optional[np.ndarray]
|
513 |
None, # image1: Optional[np.ndarray]
|
ui/config.yaml
CHANGED
@@ -41,6 +41,7 @@ matcher_zoo:
|
|
41 |
DUSt3R:
|
42 |
# TODO: duster is under development
|
43 |
enable: true
|
|
|
44 |
matcher: duster
|
45 |
dense: true
|
46 |
info:
|
@@ -52,6 +53,7 @@ matcher_zoo:
|
|
52 |
display: true
|
53 |
GIM(dkm):
|
54 |
enable: true
|
|
|
55 |
matcher: gim(dkm)
|
56 |
dense: true
|
57 |
info:
|
@@ -63,6 +65,7 @@ matcher_zoo:
|
|
63 |
display: true
|
64 |
RoMa:
|
65 |
matcher: roma
|
|
|
66 |
dense: true
|
67 |
info:
|
68 |
name: RoMa #dispaly name
|
@@ -73,6 +76,7 @@ matcher_zoo:
|
|
73 |
display: true
|
74 |
dkm:
|
75 |
matcher: dkm
|
|
|
76 |
dense: true
|
77 |
info:
|
78 |
name: DKM #dispaly name
|
@@ -398,9 +402,9 @@ matcher_zoo:
|
|
398 |
display: true
|
399 |
|
400 |
sfd2+imp:
|
|
|
401 |
matcher: imp
|
402 |
feature: sfd2
|
403 |
-
enable: true
|
404 |
dense: false
|
405 |
info:
|
406 |
name: SFD2+IMP #dispaly name
|
@@ -411,9 +415,9 @@ matcher_zoo:
|
|
411 |
display: true
|
412 |
|
413 |
sfd2+mnn:
|
|
|
414 |
matcher: NN-mutual
|
415 |
feature: sfd2
|
416 |
-
enable: true
|
417 |
dense: false
|
418 |
info:
|
419 |
name: SFD2+MNN #dispaly name
|
|
|
41 |
DUSt3R:
|
42 |
# TODO: duster is under development
|
43 |
enable: true
|
44 |
+
# skip_ci: true
|
45 |
matcher: duster
|
46 |
dense: true
|
47 |
info:
|
|
|
53 |
display: true
|
54 |
GIM(dkm):
|
55 |
enable: true
|
56 |
+
# skip_ci: true
|
57 |
matcher: gim(dkm)
|
58 |
dense: true
|
59 |
info:
|
|
|
65 |
display: true
|
66 |
RoMa:
|
67 |
matcher: roma
|
68 |
+
skip_ci: true
|
69 |
dense: true
|
70 |
info:
|
71 |
name: RoMa #dispaly name
|
|
|
76 |
display: true
|
77 |
dkm:
|
78 |
matcher: dkm
|
79 |
+
skip_ci: true
|
80 |
dense: true
|
81 |
info:
|
82 |
name: DKM #dispaly name
|
|
|
402 |
display: true
|
403 |
|
404 |
sfd2+imp:
|
405 |
+
enable: true
|
406 |
matcher: imp
|
407 |
feature: sfd2
|
|
|
408 |
dense: false
|
409 |
info:
|
410 |
name: SFD2+IMP #dispaly name
|
|
|
415 |
display: true
|
416 |
|
417 |
sfd2+mnn:
|
418 |
+
enable: true
|
419 |
matcher: NN-mutual
|
420 |
feature: sfd2
|
|
|
421 |
dense: false
|
422 |
info:
|
423 |
name: SFD2+MNN #dispaly name
|
ui/sfm.py
CHANGED
@@ -1,9 +1,10 @@
|
|
1 |
import shutil
|
|
|
2 |
import tempfile
|
3 |
from pathlib import Path
|
4 |
from typing import Any, Dict, List
|
5 |
|
6 |
-
|
7 |
|
8 |
from hloc import (
|
9 |
extract_features,
|
@@ -14,7 +15,12 @@ from hloc import (
|
|
14 |
visualization,
|
15 |
)
|
16 |
|
17 |
-
|
|
|
|
|
|
|
|
|
|
|
18 |
|
19 |
|
20 |
class SfmEngine:
|
|
|
1 |
import shutil
|
2 |
+
import sys
|
3 |
import tempfile
|
4 |
from pathlib import Path
|
5 |
from typing import Any, Dict, List
|
6 |
|
7 |
+
sys.path.append(str(Path(__file__).parents[1]))
|
8 |
|
9 |
from hloc import (
|
10 |
extract_features,
|
|
|
15 |
visualization,
|
16 |
)
|
17 |
|
18 |
+
try:
|
19 |
+
import pycolmap
|
20 |
+
except ImportError:
|
21 |
+
logger.warning("pycolmap not installed, some features may not work")
|
22 |
+
|
23 |
+
from ui.viz import fig2im
|
24 |
|
25 |
|
26 |
class SfmEngine:
|
ui/utils.py
CHANGED
@@ -2,6 +2,7 @@ import os
|
|
2 |
import pickle
|
3 |
import random
|
4 |
import shutil
|
|
|
5 |
import time
|
6 |
import warnings
|
7 |
from itertools import combinations
|
@@ -16,6 +17,8 @@ import poselib
|
|
16 |
import psutil
|
17 |
from PIL import Image
|
18 |
|
|
|
|
|
19 |
from hloc import (
|
20 |
DEVICE,
|
21 |
extract_features,
|
@@ -26,8 +29,7 @@ from hloc import (
|
|
26 |
matchers,
|
27 |
)
|
28 |
from hloc.utils.base_model import dynamic_load
|
29 |
-
|
30 |
-
from .viz import display_keypoints, display_matches, fig2im, plot_images
|
31 |
|
32 |
warnings.simplefilter("ignore")
|
33 |
|
|
|
2 |
import pickle
|
3 |
import random
|
4 |
import shutil
|
5 |
+
import sys
|
6 |
import time
|
7 |
import warnings
|
8 |
from itertools import combinations
|
|
|
17 |
import psutil
|
18 |
from PIL import Image
|
19 |
|
20 |
+
sys.path.append(str(Path(__file__).parents[1]))
|
21 |
+
|
22 |
from hloc import (
|
23 |
DEVICE,
|
24 |
extract_features,
|
|
|
29 |
matchers,
|
30 |
)
|
31 |
from hloc.utils.base_model import dynamic_load
|
32 |
+
from ui.viz import display_keypoints, display_matches, fig2im, plot_images
|
|
|
33 |
|
34 |
warnings.simplefilter("ignore")
|
35 |
|
ui/viz.py
CHANGED
@@ -1,3 +1,4 @@
|
|
|
|
1 |
import typing
|
2 |
from pathlib import Path
|
3 |
from typing import Dict, List, Optional, Tuple, Union
|
@@ -8,6 +9,8 @@ import matplotlib.pyplot as plt
|
|
8 |
import numpy as np
|
9 |
import seaborn as sns
|
10 |
|
|
|
|
|
11 |
from hloc.utils.viz import add_text, plot_keypoints
|
12 |
|
13 |
np.random.seed(1995)
|
|
|
1 |
+
import sys
|
2 |
import typing
|
3 |
from pathlib import Path
|
4 |
from typing import Dict, List, Optional, Tuple, Union
|
|
|
9 |
import numpy as np
|
10 |
import seaborn as sns
|
11 |
|
12 |
+
sys.path.append(str(Path(__file__).parents[1]))
|
13 |
+
|
14 |
from hloc.utils.viz import add_text, plot_keypoints
|
15 |
|
16 |
np.random.seed(1995)
|