diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..c7054750437d59592fda3931d1ce242d73b0568c 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +gradio-demo-img.png filter=lfs diff=lfs merge=lfs -text diff --git a/.github/workflows/update_space.yml b/.github/workflows/update_space.yml new file mode 100644 index 0000000000000000000000000000000000000000..67dbc84e4e59320a7c98b94460eb976e5cd2984f --- /dev/null +++ b/.github/workflows/update_space.yml @@ -0,0 +1,28 @@ +name: Run Python script + +on: + push: + branches: + - main + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.9' + + - name: Install Gradio + run: python -m pip install gradio + + - name: Log in to Hugging Face + run: python -c 'import huggingface_hub; huggingface_hub.login(token="${{ secrets.hf_token }}")' + + - name: Deploy to Spaces + run: gradio deploy diff --git a/README.md b/README.md index e82729f45cb9756337cb6a528639ee45e5bbcc7f..843c6b643fe62ff99fa93ba90002566aa0406431 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,7 @@ --- -title: CountGD Multi-Modal Open-World Counting -emoji: 🏃 -colorFrom: blue -colorTo: purple +title: CountGD_Multi-Modal_Open-World_Counting +app_file: app.py sdk: gradio sdk_version: 4.37.2 -app_file: app.py -pinned: false --- - -Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference +nohup python -u app.py & diff --git a/app.log b/app.log new file mode 100644 index 0000000000000000000000000000000000000000..af6a9c7ad6d4f319f1494008c38d2d7774988f9b --- /dev/null +++ b/app.log @@ -0,0 +1,15 @@ +nohup: ignoring input +/home/niki/anaconda3/envs/gradio/lib/python3.9/site-packages/torch/functional.py:507: UserWarning: torch.meshgrid: in an upcoming release, it will be required to pass the indexing argument. (Triggered internally at ../aten/src/ATen/native/TensorShape.cpp:3549.) + return _VF.meshgrid(tensors, **kwargs) # type: ignore[attr-defined] +Some weights of BertModel were not initialized from the model checkpoint at checkpoints/bert-base-uncased and are newly initialized: ['pooler.dense.bias', 'pooler.dense.weight'] +You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference. +huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks... +To disable this warning, you can either: + - Avoid using `tokenizers` before the fork if possible + - Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false) +/home/niki/anaconda3/envs/gradio/lib/python3.9/site-packages/transformers/modeling_utils.py:977: FutureWarning: The `device` argument is deprecated and will be removed in v5 of Transformers. + warnings.warn( +/home/niki/anaconda3/envs/gradio/lib/python3.9/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/home/niki/anaconda3/envs/gradio/lib/python3.9/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None + warnings.warn( diff --git a/app.py b/app.py new file mode 100644 index 0000000000000000000000000000000000000000..b1108a841536962b6b657df563cdd925f90f89ba --- /dev/null +++ b/app.py @@ -0,0 +1,437 @@ +import gradio as gr +import copy +import random +import torch +import PIL +from PIL import Image, ImageDraw, ImageFont +import torchvision.transforms.functional as F +import numpy as np +import argparse +import json +import plotly.express as px +import pandas as pd +from util.slconfig import SLConfig, DictAction +from util.misc import nested_tensor_from_tensor_list +import datasets.transforms as T +import scipy.ndimage as ndimage +import matplotlib.pyplot as plt +from gradio_image_prompter import ImagePrompter +# https://github.com/PhyscalX/gradio-image-prompter/tree/main/backend/gradio_image_prompter/templates/component +import io +from enum import Enum +import os +os.environ["GRADIO_TEMP_DIR"] = os.path.join(os.getcwd(), "tmp") + +class AppSteps(Enum): + JUST_TEXT = 1 + TEXT_AND_EXEMPLARS = 2 + JUST_EXEMPLARS = 3 + FULL_APP = 4 + +CONF_THRESH = 0.23 + +# MODEL: +def get_args_parser(): + """ + Example eval command: + + >> python main.py --output_dir ./gdino_test -c config/cfg_fsc147_vit_b_test.py --eval --datasets config/datasets_fsc147.json --pretrain_model_path ../checkpoints_and_logs/gdino_train/checkpoint_best_regular.pth --options text_encoder_type=checkpoints/bert-base-uncased --sam_tt_norm --crop + """ + parser = argparse.ArgumentParser("Set transformer detector", add_help=False) + parser.add_argument( + "--options", + nargs="+", + action=DictAction, + help="override some settings in the used config, the key-value pair " + "in xxx=yyy format will be merged into config file.", + ) + + # dataset parameters + parser.add_argument("--remove_difficult", action="store_true") + parser.add_argument("--fix_size", action="store_true") + + # training parameters + parser.add_argument("--note", default="", help="add some notes to the experiment") + parser.add_argument( + "--device", default="cuda", help="device to use for training / testing" + ) + parser.add_argument("--resume", default="", help="resume from checkpoint") + parser.add_argument( + "--pretrain_model_path", + help="load from other checkpoint", + default="checkpoint_best_regular.pth", + ) + parser.add_argument("--finetune_ignore", type=str, nargs="+") + parser.add_argument( + "--start_epoch", default=0, type=int, metavar="N", help="start epoch" + ) + parser.add_argument("--eval", action="store_false") + parser.add_argument("--num_workers", default=8, type=int) + parser.add_argument("--test", action="store_true") + parser.add_argument("--debug", action="store_true") + parser.add_argument("--find_unused_params", action="store_true") + parser.add_argument("--save_results", action="store_true") + parser.add_argument("--save_log", action="store_true") + + # distributed training parameters + parser.add_argument( + "--world_size", default=1, type=int, help="number of distributed processes" + ) + parser.add_argument( + "--dist_url", default="env://", help="url used to set up distributed training" + ) + parser.add_argument( + "--rank", default=0, type=int, help="number of distributed processes" + ) + parser.add_argument( + "--local_rank", type=int, help="local rank for DistributedDataParallel" + ) + parser.add_argument( + "--local-rank", type=int, help="local rank for DistributedDataParallel" + ) + parser.add_argument("--amp", action="store_true", help="Train with mixed precision") + return parser + +# Get counting model. +def build_model_and_transforms(args): + normalize = T.Compose( + [T.ToTensor(), T.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])] + ) + data_transform = T.Compose( + [ + T.RandomResize([800], max_size=1333), + normalize, + ] + ) + cfg = SLConfig.fromfile("cfg_app.py") + cfg.merge_from_dict({"text_encoder_type": "checkpoints/bert-base-uncased"}) + cfg_dict = cfg._cfg_dict.to_dict() + args_vars = vars(args) + for k, v in cfg_dict.items(): + if k not in args_vars: + setattr(args, k, v) + else: + raise ValueError("Key {} can used by args only".format(k)) + + device = torch.device(args.device) + # fix the seed for reproducibility + seed = 42 + torch.manual_seed(seed) + np.random.seed(seed) + random.seed(seed) + + # we use register to maintain models from catdet6 on. + from models.registry import MODULE_BUILD_FUNCS + + assert args.modelname in MODULE_BUILD_FUNCS._module_dict + + build_func = MODULE_BUILD_FUNCS.get(args.modelname) + model, _, _ = build_func(args) + + model.to(device) + + checkpoint = torch.load(args.pretrain_model_path, map_location="cpu")["model"] + model.load_state_dict(checkpoint, strict=False) + + model.eval() + + return model, data_transform + + +parser = argparse.ArgumentParser("Counting Application", parents=[get_args_parser()]) +args = parser.parse_args() +model, transform = build_model_and_transforms(args) + +examples = [ + ["strawberry.jpg", "strawberry", {"image": "strawberry.jpg"}], + ["strawberry.jpg", "blueberry", {"image": "strawberry.jpg"}], + ["bird-1.JPG", "bird", {"image": "bird-2.JPG"}], + ["fish.jpg", "fish", {"image": "fish.jpg"}], + ["women.jpg", "girl", {"image": "women.jpg"}], + ["women.jpg", "boy", {"image": "women.jpg"}], + ["balloon.jpg", "hot air balloon", {"image": "balloon.jpg"}], + ["deer.jpg", "deer", {"image": "deer.jpg"}], + ["apple.jpg", "apple", {"image": "apple.jpg"}], + ["egg.jpg", "egg", {"image": "egg.jpg"}], + ["stamp.jpg", "stamp", {"image": "stamp.jpg"}], + ["green-pea.jpg", "green pea", {"image": "green-pea.jpg"}], + ["lego.jpg", "lego", {"image": "lego.jpg"}] +] + +# APP: +def get_box_inputs(prompts): + box_inputs = [] + for prompt in prompts: + if prompt[2] == 2.0 and prompt[5] == 3.0: + box_inputs.append([prompt[0], prompt[1], prompt[3], prompt[4]]) + + return box_inputs + +def get_ind_to_filter(text, word_ids, keywords): + if len(keywords) <= 0: + return list(range(len(word_ids))) + input_words = text.split() + keywords = keywords.split(",") + keywords = [keyword.strip() for keyword in keywords] + + word_inds = [] + for keyword in keywords: + if keyword in input_words: + if len(word_inds) <= 0: + ind = input_words.index(keyword) + word_inds.append(ind) + else: + ind = input_words.index(keyword, word_inds[-1]) + word_inds.append(ind) + else: + raise Exception("Only specify keywords in the input text!") + + inds_to_filter = [] + for ind in range(len(word_ids)): + word_id = word_ids[ind] + if word_id in word_inds: + inds_to_filter.append(ind) + + return inds_to_filter + +def count(image, text, prompts, state): + print("state: " + str(state)) + keywords = "" # do not handle this for now + # Handle no prompt case. + if prompts is None: + prompts = {"image": image, "points": []} + input_image, _ = transform(image, {"exemplars": torch.tensor([])}) + input_image = input_image.unsqueeze(0).cuda() + exemplars = get_box_inputs(prompts["points"]) + print(exemplars) + input_image_exemplars, exemplars = transform(prompts["image"], {"exemplars": torch.tensor(exemplars)}) + input_image_exemplars = input_image_exemplars.unsqueeze(0).cuda() + exemplars = [exemplars["exemplars"].cuda()] + + with torch.no_grad(): + model_output = model( + nested_tensor_from_tensor_list(input_image), + nested_tensor_from_tensor_list(input_image_exemplars), + exemplars, + [torch.tensor([0]).cuda() for _ in range(len(input_image))], + captions=[text + " ."] * len(input_image), + ) + + ind_to_filter = get_ind_to_filter(text, model_output["token"][0].word_ids, keywords) + print(model_output["token"][0].tokens) + print(ind_to_filter) + print(model_output["pred_logits"].sigmoid()[0].shape) + logits = model_output["pred_logits"].sigmoid()[0][:, ind_to_filter] + print(logits.shape) + boxes = model_output["pred_boxes"][0] + if len(keywords.strip()) > 0: + box_mask = (logits > CONF_THRESH).sum(dim=-1) == len(ind_to_filter) + else: + box_mask = logits.max(dim=-1).values > CONF_THRESH + logits = logits[box_mask, :].cpu().numpy() + boxes = boxes[box_mask, :].cpu().numpy() + + # Plot results. + (w, h) = image.size + det_map = np.zeros((h, w)) + det_map[(h * boxes[:, 1]).astype(int), (w * boxes[:, 0]).astype(int)] = 1 + det_map = ndimage.gaussian_filter( + det_map, sigma=(w // 200, w // 200), order=0 + ) + plt.imshow(image) + plt.imshow(det_map[None, :].transpose(1, 2, 0), 'jet', interpolation='none', alpha=0.7) + plt.axis('off') + img_buf = io.BytesIO() + plt.savefig(img_buf, format='png', bbox_inches='tight') + + output_img = Image.open(img_buf) + + if AppSteps.TEXT_AND_EXEMPLARS not in state: + exemplar_image = ImagePrompter(type='pil', label='Visual Exemplar Image', value=prompts, interactive=True, visible=True) + new_submit_btn = gr.Button("Count", variant="primary", interactive=False) + state = [AppSteps.JUST_TEXT, AppSteps.TEXT_AND_EXEMPLARS] + main_instructions_comp = gr.Markdown(visible=False) + step_3 = gr.Tab(visible=False) + elif AppSteps.FULL_APP not in state: + exemplar_image = ImagePrompter(type='pil', label='Visual Exemplar Image', value=prompts, interactive=True, visible=True) + new_submit_btn = submit_btn + state = [AppSteps.JUST_TEXT, AppSteps.TEXT_AND_EXEMPLARS, AppSteps.FULL_APP] + main_instructions_comp = gr.Markdown(visible=True) + step_3 = gr.Tab(visible=True) + else: + exemplar_image = ImagePrompter(type='pil', label='Visual Exemplar Image', value=prompts, interactive=True, visible=True) + new_submit_btn = submit_btn + main_instructions_comp = gr.Markdown(visible=True) + step_3 = gr.Tab(visible=True) + + out_label = "Detected instances predicted with" + if len(text.strip()) > 0: + out_label += " text" + if exemplars[0].size()[0] == 1: + out_label += " and " + str(exemplars[0].size()[0]) + " visual exemplar." + elif exemplars[0].size()[0] > 1: + out_label += " and " + str(exemplars[0].size()[0]) + " visual exemplars." + else: + out_label += "." + elif exemplars[0].size()[0] > 0: + if exemplars[0].size()[0] == 1: + out_label += " " + str(exemplars[0].size()[0]) + " visual exemplar." + else: + out_label += " " + str(exemplars[0].size()[0]) + " visual exemplars." + else: + out_label = "Nothing specified to detect." + return (gr.Image(output_img, visible=True, label=out_label, show_label=True), gr.Number(label="Predicted Count", visible=True, value=boxes.shape[0]), new_submit_btn, gr.Tab(visible=True), step_3, state) + +def count_main(image, text, prompts): + keywords = "" # do not handle this for now + # Handle no prompt case. + if prompts is None: + prompts = {"image": image, "points": []} + input_image, _ = transform(image, {"exemplars": torch.tensor([])}) + input_image = input_image.unsqueeze(0).cuda() + exemplars = get_box_inputs(prompts["points"]) + print(exemplars) + input_image_exemplars, exemplars = transform(prompts["image"], {"exemplars": torch.tensor(exemplars)}) + input_image_exemplars = input_image_exemplars.unsqueeze(0).cuda() + exemplars = [exemplars["exemplars"].cuda()] + + with torch.no_grad(): + model_output = model( + nested_tensor_from_tensor_list(input_image), + nested_tensor_from_tensor_list(input_image_exemplars), + exemplars, + [torch.tensor([0]).cuda() for _ in range(len(input_image))], + captions=[text + " ."] * len(input_image), + ) + + ind_to_filter = get_ind_to_filter(text, model_output["token"][0].word_ids, keywords) + print(model_output["token"][0].tokens) + print(ind_to_filter) + print(model_output["pred_logits"].sigmoid()[0].shape) + logits = model_output["pred_logits"].sigmoid()[0][:, ind_to_filter] + print(logits.shape) + boxes = model_output["pred_boxes"][0] + if len(keywords.strip()) > 0: + box_mask = (logits > CONF_THRESH).sum(dim=-1) == len(ind_to_filter) + else: + box_mask = logits.max(dim=-1).values > CONF_THRESH + logits = logits[box_mask, :].cpu().numpy() + boxes = boxes[box_mask, :].cpu().numpy() + + # Plot results. + (w, h) = image.size + det_map = np.zeros((h, w)) + det_map[(h * boxes[:, 1]).astype(int), (w * boxes[:, 0]).astype(int)] = 1 + det_map = ndimage.gaussian_filter( + det_map, sigma=(w // 200, w // 200), order=0 + ) + plt.imshow(image) + plt.imshow(det_map[None, :].transpose(1, 2, 0), 'jet', interpolation='none', alpha=0.7) + plt.axis('off') + img_buf = io.BytesIO() + plt.savefig(img_buf, format='png', bbox_inches='tight') + + output_img = Image.open(img_buf) + + out_label = "Detected instances predicted with" + if len(text.strip()) > 0: + out_label += " text" + if exemplars[0].size()[0] == 1: + out_label += " and " + str(exemplars[0].size()[0]) + " visual exemplar." + elif exemplars[0].size()[0] > 1: + out_label += " and " + str(exemplars[0].size()[0]) + " visual exemplars." + else: + out_label += "." + elif exemplars[0].size()[0] > 0: + if exemplars[0].size()[0] == 1: + out_label += " " + str(exemplars[0].size()[0]) + " visual exemplar." + else: + out_label += " " + str(exemplars[0].size()[0]) + " visual exemplars." + else: + out_label = "Nothing specified to detect." + return (gr.Image(output_img, visible=True, label=out_label, show_label=True), gr.Number(label="Predicted Count", visible=True, value=boxes.shape[0])) + +def remove_label(image): + return gr.Image(show_label=False) + +def check_submit_btn(exemplar_image_prompts, state): + if AppSteps.TEXT_AND_EXEMPLARS not in state or len(state) == 3: + return gr.Button("Count", variant="primary", interactive=True) + elif exemplar_image_prompts is None: + return gr.Button("Count", variant="primary", interactive=False) + elif len(get_box_inputs(exemplar_image_prompts["points"])) > 0: + return gr.Button("Count", variant="primary", interactive=True) + else: + return gr.Button("Count", variant="primary", interactive=False) + +exemplar_img_drawing_instructions_part_1 = '

Congrats, you have counted the strawberries! You can also draw a box around the object you want to count. Click and drag the mouse on the image below to draw a box around one of the strawberries. You can click the back button in the top right of the image to delete the box and try again.

' +exemplar_img_drawing_instructions_part_2 = '

The boxes you draw are called \"visual exemplars,\" image examples of what you want the model to count. You can add more boxes around more examples of strawberries in the image above to increase the accuracy of the predicted count. You can also use strawberries from a different image to specify the object to count by uploading or pasting a new image above and drawing boxes around strawberries in it.

' +instructions_main = """ +# How to Use the App +As shown earlier, there are 3 ways to specify the object to count: (1) with text only, (2) with text and any number of boxes (i.e., "visual exemplars") around example objects, and (3) with visual exemplars only. What is being used is indicated in the top left of the output image. How to try each case is detailed below. + +
    +
  1. Text Only: Only provide text describing the object to count in the textbox titled "What would you like to count?" Delete all boxes drawn on the visual exemplar image.
    2. +
  2. Text + Visual Exemplars: Provide text describing the object to count in the textbox titled "What would you like to count?" and draw at least one box around an example object in the visual exemplar image.
    3. +
  3. Visual Exemplars Only: Remove all text in the textbox titled "What would you like to count?" and draw at least one box around an example object in the visual exemplar image.
    4. +
+ +## Click on the "App" tab at the top of the screen to exit the tutorial and start using the main app! +""" + +with gr.Blocks(title="CountGD: Multi-Modal Open-World Counting", theme="soft", head="""""") as demo: + state = gr.State(value=[AppSteps.JUST_TEXT]) + with gr.Tab("Tutorial"): + with gr.Row(): + with gr.Column(): + with gr.Tab("Step 3", visible=False) as step_3: + main_instructions = gr.Markdown(instructions_main) + with gr.Tab("Step 2", visible=False) as step_2: + gr.Markdown(exemplar_img_drawing_instructions_part_1) + exemplar_image = ImagePrompter(type='pil', label='Visual Exemplar Image', show_label=True, value={"image": "strawberry.jpg", "points": []}, interactive=True) + with gr.Accordion("Open for Further Information", open=False): + gr.Markdown(exemplar_img_drawing_instructions_part_2) + with gr.Tab("Step 1", visible=True) as step_1: + input_image = gr.Image(type='pil', label='Input Image', show_label='True', value="strawberry.jpg", interactive=False, width="30vw") + gr.Markdown('# Click "Count" to count the strawberries.') + + with gr.Column(): + with gr.Tab("Output Image"): + detected_instances = gr.Image(label="Detected Instances", show_label='True', interactive=False, visible=True, width="40vw") + + with gr.Row(): + input_text = gr.Textbox(label="What would you like to count?", value="strawberry", interactive=True) + pred_count = gr.Number(label="Predicted Count", visible=False) + submit_btn = gr.Button("Count", variant="primary", interactive=True) + + submit_btn.click(fn=remove_label, inputs=[detected_instances], outputs=[detected_instances]).then(fn=count, inputs=[input_image, input_text, exemplar_image, state], outputs=[detected_instances, pred_count, submit_btn, step_2, step_3, state]) + exemplar_image.change(check_submit_btn, inputs=[exemplar_image, state], outputs=[submit_btn]) + with gr.Tab("App", visible=True) as main_app: + + gr.Markdown( + """ + #
CountGD: Multi-Modal Open-World Counting +

Count objects with text, visual exemplars, or both together.

+

Scroll down to try more examples

+

[paper] + [code]

+ Limitation: this app does not support fine-grained counting based on attributes or visual grounding inputs yet.
+ """ + ) + + with gr.Row(): + with gr.Column(): + input_image_main = gr.Image(type='pil', label='Input Image', show_label='True', value="strawberry.jpg", interactive=True) + input_text_main = gr.Textbox(label="What would you like to count?", placeholder="", value="strawberry") + exemplar_image_main = ImagePrompter(type='pil', label='Visual Exemplar Image', show_label=True, value={"image": "strawberry.jpg", "points": []}, interactive=True) + with gr.Column(): + detected_instances_main = gr.Image(label="Detected Instances", show_label='True', interactive=False) + pred_count_main = gr.Number(label="Predicted Count") + submit_btn_main = gr.Button("Count", variant="primary") + clear_btn_main = gr.ClearButton(variant="secondary") + gr.Examples(label="Examples: click on a row to load the example. Add visual exemplars by drawing boxes on the loaded \"Visual Exemplar Image.\"", examples=examples, inputs=[input_image_main, input_text_main, exemplar_image_main]) + submit_btn_main.click(fn=remove_label, inputs=[detected_instances_main], outputs=[detected_instances_main]).then(fn=count_main, inputs=[input_image_main, input_text_main, exemplar_image_main], outputs=[detected_instances_main, pred_count_main]) + clear_btn_main.add([input_image_main, input_text_main, exemplar_image_main, detected_instances_main, pred_count_main]) + + +demo.launch(share=True, allowed_paths=['back-icon.jpg', 'paste-icon.jpg', 'upload-icon.jpg', 'button-legend.jpg']) diff --git a/apple.jpg b/apple.jpg new file mode 100644 index 0000000000000000000000000000000000000000..993cc2d709946bc68cd13c87854e2d3c0908ad2f Binary files /dev/null and b/apple.jpg differ diff --git a/back-icon.jpg b/back-icon.jpg new file mode 100644 index 0000000000000000000000000000000000000000..4a0933ac4b0f94a03992bd4227638459607a11e0 Binary files /dev/null and b/back-icon.jpg differ diff --git a/balloon.jpg b/balloon.jpg new file mode 100644 index 0000000000000000000000000000000000000000..b492dd11a01260ec9693e6e251100a6eec657bd9 Binary files /dev/null and b/balloon.jpg differ diff --git a/bird-1.JPG b/bird-1.JPG new file mode 100644 index 0000000000000000000000000000000000000000..6975666d7dd574cdcd519601b4460f015353e632 Binary files /dev/null and b/bird-1.JPG differ diff --git a/bird-2.JPG b/bird-2.JPG new file mode 100644 index 0000000000000000000000000000000000000000..af61947e18fc81a225f906a1479febdc074ccfc2 Binary files /dev/null and b/bird-2.JPG differ diff --git a/button-legend.jpg b/button-legend.jpg new file mode 100644 index 0000000000000000000000000000000000000000..624841b3c58a571f80b959519fd35f5be8ba0b0c Binary files /dev/null and b/button-legend.jpg differ diff --git a/cfg_app.py b/cfg_app.py new file mode 100644 index 0000000000000000000000000000000000000000..83332206e779e375d9a48178a4efbcca94217358 --- /dev/null +++ b/cfg_app.py @@ -0,0 +1,118 @@ +data_aug_scales = [480, 512, 544, 576, 608, 640, 672, 704, 736, 768, 800] +data_aug_max_size = 1333 +data_aug_scales2_resize = [400, 500, 600] +data_aug_scales2_crop = [384, 600] +data_aug_scale_overlap = None +batch_size = 4 +modelname = 'groundingdino' +backbone = "swin_B_384_22k" +position_embedding = 'sine' +pe_temperatureH = 20 +pe_temperatureW = 20 +return_interm_indices = [1, 2, 3] +enc_layers = 6 +dec_layers = 6 +pre_norm = False +dim_feedforward = 2048 +hidden_dim = 256 +dropout = 0.0 +nheads = 8 +num_queries = 900 +query_dim = 4 +num_patterns = 0 +num_feature_levels = 4 +enc_n_points = 4 +dec_n_points = 4 +two_stage_type = 'standard' +two_stage_bbox_embed_share = False +two_stage_class_embed_share = False +transformer_activation = 'relu' +dec_pred_bbox_embed_share = True +dn_box_noise_scale = 1.0 +dn_label_noise_ratio = 0.5 +dn_label_coef = 1.0 +dn_bbox_coef = 1.0 +embed_init_tgt = True +dn_labelbook_size = 91 +max_text_len = 256 +text_encoder_type = "bert-base-uncased" +use_text_enhancer = True +use_fusion_layer = True +use_checkpoint = True +use_transformer_ckpt = True +use_text_cross_attention = True +text_dropout = 0.0 +fusion_dropout = 0.0 +fusion_droppath = 0.1 +sub_sentence_present = True +max_labels = 90 # pos + neg +lr = 0.0001 # base learning rate +backbone_freeze_keywords = None # only for gdino backbone +freeze_keywords = ['backbone.0', 'bert'] # for whole model, e.g. ['backbone.0', 'bert'] for freeze visual encoder and text encoder +lr_backbone = 1e-05 # specific learning rate +lr_backbone_names = ['backbone.0', 'bert'] +lr_linear_proj_mult = 1e-05 +lr_linear_proj_names = ['ref_point_head', 'sampling_offsets'] +weight_decay = 0.0001 +param_dict_type = 'ddetr_in_mmdet' +ddetr_lr_param = False +epochs = 30 +lr_drop = 10 +save_checkpoint_interval = 10 +clip_max_norm = 0.1 +onecyclelr = False +multi_step_lr = False +lr_drop_list = [10, 20] +frozen_weights = None +dilation = False +pdetr3_bbox_embed_diff_each_layer = False +pdetr3_refHW = -1 +random_refpoints_xy = False +fix_refpoints_hw = -1 +dabdetr_yolo_like_anchor_update = False +dabdetr_deformable_encoder = False +dabdetr_deformable_decoder = False +use_deformable_box_attn = False +box_attn_type = 'roi_align' +dec_layer_number = None +decoder_layer_noise = False +dln_xy_noise = 0.2 +dln_hw_noise = 0.2 +add_channel_attention = False +add_pos_value = False +two_stage_pat_embed = 0 +two_stage_add_query_num = 0 +two_stage_learn_wh = False +two_stage_default_hw = 0.05 +two_stage_keep_all_tokens = False +num_select = 900 +batch_norm_type = 'FrozenBatchNorm2d' +masks = False +aux_loss = True +set_cost_class = 5.0 +set_cost_bbox = 1.0 +set_cost_giou = 0.0 +cls_loss_coef = 5.0 +bbox_loss_coef = 1.0 +giou_loss_coef = 0.0 +enc_loss_coef = 1.0 +interm_loss_coef = 1.0 +no_interm_box_loss = False +mask_loss_coef = 1.0 +dice_loss_coef = 1.0 +focal_alpha = 0.25 +focal_gamma = 2.0 +decoder_sa_type = 'sa' +matcher_type = 'HungarianMatcher' +decoder_module_seq = ['sa', 'ca', 'ffn'] +nms_iou_threshold = -1 +dec_pred_class_embed_share = True +match_unstable_error = True +use_detached_boxes_dec_out = False +dn_scalar = 100 + +box_threshold = 0.23 +text_threshold = 0 +use_coco_eval = False +label_list = ['alcohol bottle', 'baguette roll', 'ball', 'banana', 'bead', 'bee', 'birthday candle', 'biscuit', 'boat', 'bottle', 'bowl', 'box', 'bread roll', 'brick', 'buffalo', 'bun', 'calamari ring', 'can', 'candle', 'cap', 'car', 'cartridge', 'cassette', 'cement bag', 'cereal', 'chewing gum piece', 'chopstick', 'clam', 'coffee bean', 'coin', 'cotton ball', 'cow', 'crane', 'crayon', 'croissant', 'crow', 'cup', 'cupcake', 'cupcake holder', 'fish', 'gemstone', 'go game piece', 'goat', 'goldfish snack', 'goose', 'ice cream', 'ice cream cone', 'instant noodle', 'jade stone', 'jeans', 'kidney bean', 'kitchen towel', 'lighter', 'lipstick', 'm&m piece', 'macaron', 'match', 'meat skewer', 'mini blind', 'mosaic tile', 'naan bread', 'nail', 'nut', 'onion ring', 'orange', 'pearl', 'pen', 'pencil', 'penguin', 'pepper', 'person', 'pigeon', 'plate', 'polka dot tile', 'potato', 'rice bag', 'roof tile', 'screw', 'shoe', 'spoon', 'spring roll', 'stair', 'stapler pin', 'straw', 'supermarket shelf', 'swan', 'tomato', 'watermelon', 'window', 'zebra'] +val_label_list = ["apple", "candy piece", "carrom board piece", "cashew nut", "comic book", "crab cake", "deer", "egg", "elephant", "finger food", "green pea", "hot air balloon", "keyboard key", "lego", "marble", "marker", "nail polish", "potato chip", "red bean", "round dessert", "sauce bottle", "sea shell", "sheep", "ski", "stamp", "sticky note", "strawberry", "sunglasses", "tree log", "watch"] diff --git a/checkpoint_best_regular.pth b/checkpoint_best_regular.pth new file mode 100644 index 0000000000000000000000000000000000000000..d4e38fb1f8646d6dd6726eef24d6f067c95dd849 --- /dev/null +++ b/checkpoint_best_regular.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c1bab864b17db345b4c6e3aaabb5765bc2c0a90d0bc8defb5e664a74a50aa126 +size 1250122522 diff --git a/checkpoints/bert-base-uncased/config.json b/checkpoints/bert-base-uncased/config.json new file mode 100644 index 0000000000000000000000000000000000000000..951186f303941fb6bd7d218c62838f5ce7e86ef1 --- /dev/null +++ b/checkpoints/bert-base-uncased/config.json @@ -0,0 +1,26 @@ +{ + "_name_or_path": "bert-base-uncased", + "architectures": [ + "BertModel" + ], + "attention_probs_dropout_prob": 0.1, + "classifier_dropout": null, + "gradient_checkpointing": false, + "hidden_act": "gelu", + "hidden_dropout_prob": 0.1, + "hidden_size": 768, + "initializer_range": 0.02, + "intermediate_size": 3072, + "layer_norm_eps": 1e-12, + "max_position_embeddings": 512, + "model_type": "bert", + "num_attention_heads": 12, + "num_hidden_layers": 12, + "pad_token_id": 0, + "position_embedding_type": "absolute", + "torch_dtype": "float32", + "transformers_version": "4.39.1", + "type_vocab_size": 2, + "use_cache": true, + "vocab_size": 30522 +} diff --git a/checkpoints/bert-base-uncased/model.safetensors b/checkpoints/bert-base-uncased/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..ce0d2625d07df50ff214ceec93b166e074e9fa5c --- /dev/null +++ b/checkpoints/bert-base-uncased/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5875f83030335d194f35b15a32e7f4e654aa302aa83af032a3f36d035dcaf8af +size 435588776 diff --git a/checkpoints/bert-base-uncased/special_tokens_map.json b/checkpoints/bert-base-uncased/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..a8b3208c2884c4efb86e49300fdd3dc877220cdf --- /dev/null +++ b/checkpoints/bert-base-uncased/special_tokens_map.json @@ -0,0 +1,7 @@ +{ + "cls_token": "[CLS]", + "mask_token": "[MASK]", + "pad_token": "[PAD]", + "sep_token": "[SEP]", + "unk_token": "[UNK]" +} diff --git a/checkpoints/bert-base-uncased/tokenizer.json b/checkpoints/bert-base-uncased/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..688882a79f44442ddc1f60d70334a7ff5df0fb47 --- /dev/null +++ b/checkpoints/bert-base-uncased/tokenizer.json @@ -0,0 +1,30672 @@ +{ + "version": "1.0", + "truncation": null, + "padding": null, + "added_tokens": [ + { + "id": 0, + "content": "[PAD]", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 100, + "content": "[UNK]", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 101, + "content": "[CLS]", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 102, + "content": "[SEP]", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 103, + "content": "[MASK]", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + } + ], + "normalizer": { + "type": "BertNormalizer", + "clean_text": true, + "handle_chinese_chars": true, + "strip_accents": null, + "lowercase": true + }, + "pre_tokenizer": { + "type": "BertPreTokenizer" + }, + "post_processor": { + "type": "TemplateProcessing", + "single": [ + { + "SpecialToken": { + "id": "[CLS]", + "type_id": 0 + } + }, + { + "Sequence": { + "id": "A", + "type_id": 0 + } + }, + { + "SpecialToken": { + "id": "[SEP]", + "type_id": 0 + } + } + ], + "pair": [ + { + "SpecialToken": { + "id": "[CLS]", + "type_id": 0 + } + }, + { + "Sequence": { + "id": "A", + "type_id": 0 + } + }, + { + "SpecialToken": { + "id": "[SEP]", + "type_id": 0 + } + }, + { + "Sequence": { + "id": "B", + "type_id": 1 + } + }, + { + "SpecialToken": { + "id": "[SEP]", + "type_id": 1 + } + } + ], + "special_tokens": { + "[CLS]": { + "id": "[CLS]", + "ids": [ + 101 + ], + "tokens": [ + "[CLS]" + ] + }, + "[SEP]": { + "id": "[SEP]", + "ids": [ + 102 + ], + "tokens": [ + "[SEP]" + ] + } + } + }, + "decoder": { + "type": "WordPiece", + "prefix": "##", + "cleanup": true + }, + "model": { + "type": "WordPiece", + "unk_token": "[UNK]", + "continuing_subword_prefix": "##", + "max_input_chars_per_word": 100, + "vocab": { + "[PAD]": 0, + "[unused0]": 1, + "[unused1]": 2, + "[unused2]": 3, + "[unused3]": 4, + "[unused4]": 5, + "[unused5]": 6, + "[unused6]": 7, + "[unused7]": 8, + "[unused8]": 9, + "[unused9]": 10, + "[unused10]": 11, + "[unused11]": 12, + "[unused12]": 13, + "[unused13]": 14, + "[unused14]": 15, + "[unused15]": 16, + "[unused16]": 17, + "[unused17]": 18, + "[unused18]": 19, + "[unused19]": 20, + "[unused20]": 21, + "[unused21]": 22, + "[unused22]": 23, + "[unused23]": 24, + "[unused24]": 25, + "[unused25]": 26, + "[unused26]": 27, + "[unused27]": 28, + "[unused28]": 29, + "[unused29]": 30, + "[unused30]": 31, + "[unused31]": 32, + "[unused32]": 33, + "[unused33]": 34, + "[unused34]": 35, + "[unused35]": 36, + "[unused36]": 37, + "[unused37]": 38, + "[unused38]": 39, + "[unused39]": 40, + "[unused40]": 41, + "[unused41]": 42, + "[unused42]": 43, + "[unused43]": 44, + "[unused44]": 45, + "[unused45]": 46, + "[unused46]": 47, + "[unused47]": 48, + "[unused48]": 49, + "[unused49]": 50, + "[unused50]": 51, + "[unused51]": 52, + "[unused52]": 53, + "[unused53]": 54, + "[unused54]": 55, + "[unused55]": 56, + "[unused56]": 57, + "[unused57]": 58, + "[unused58]": 59, + "[unused59]": 60, + "[unused60]": 61, + "[unused61]": 62, + "[unused62]": 63, + "[unused63]": 64, + "[unused64]": 65, + "[unused65]": 66, + "[unused66]": 67, + "[unused67]": 68, + "[unused68]": 69, + "[unused69]": 70, + "[unused70]": 71, + "[unused71]": 72, + "[unused72]": 73, + "[unused73]": 74, + "[unused74]": 75, + "[unused75]": 76, + "[unused76]": 77, + "[unused77]": 78, + "[unused78]": 79, + "[unused79]": 80, + "[unused80]": 81, + "[unused81]": 82, + "[unused82]": 83, + "[unused83]": 84, + "[unused84]": 85, + "[unused85]": 86, + "[unused86]": 87, + "[unused87]": 88, + "[unused88]": 89, + "[unused89]": 90, + "[unused90]": 91, + "[unused91]": 92, + "[unused92]": 93, + "[unused93]": 94, + "[unused94]": 95, + "[unused95]": 96, + "[unused96]": 97, + "[unused97]": 98, + "[unused98]": 99, + "[UNK]": 100, + "[CLS]": 101, + "[SEP]": 102, + "[MASK]": 103, + "[unused99]": 104, + "[unused100]": 105, + "[unused101]": 106, + "[unused102]": 107, + "[unused103]": 108, + "[unused104]": 109, + "[unused105]": 110, + "[unused106]": 111, + "[unused107]": 112, + "[unused108]": 113, + "[unused109]": 114, + "[unused110]": 115, + "[unused111]": 116, + "[unused112]": 117, + "[unused113]": 118, + "[unused114]": 119, + "[unused115]": 120, + "[unused116]": 121, + "[unused117]": 122, + "[unused118]": 123, + "[unused119]": 124, + "[unused120]": 125, + "[unused121]": 126, + "[unused122]": 127, + "[unused123]": 128, + "[unused124]": 129, + "[unused125]": 130, + "[unused126]": 131, + "[unused127]": 132, + "[unused128]": 133, + "[unused129]": 134, + "[unused130]": 135, + "[unused131]": 136, + "[unused132]": 137, + "[unused133]": 138, + "[unused134]": 139, + "[unused135]": 140, + "[unused136]": 141, + "[unused137]": 142, + "[unused138]": 143, + "[unused139]": 144, + "[unused140]": 145, + "[unused141]": 146, + "[unused142]": 147, + "[unused143]": 148, + "[unused144]": 149, + "[unused145]": 150, + "[unused146]": 151, + "[unused147]": 152, + "[unused148]": 153, + "[unused149]": 154, + "[unused150]": 155, + "[unused151]": 156, + "[unused152]": 157, + "[unused153]": 158, + "[unused154]": 159, + "[unused155]": 160, + "[unused156]": 161, + "[unused157]": 162, + "[unused158]": 163, + "[unused159]": 164, + "[unused160]": 165, + "[unused161]": 166, + "[unused162]": 167, + "[unused163]": 168, + "[unused164]": 169, + "[unused165]": 170, + "[unused166]": 171, + "[unused167]": 172, + "[unused168]": 173, + "[unused169]": 174, + "[unused170]": 175, + "[unused171]": 176, + "[unused172]": 177, + "[unused173]": 178, + "[unused174]": 179, + "[unused175]": 180, + "[unused176]": 181, + "[unused177]": 182, + "[unused178]": 183, + "[unused179]": 184, + "[unused180]": 185, + "[unused181]": 186, + "[unused182]": 187, + "[unused183]": 188, + "[unused184]": 189, + "[unused185]": 190, + "[unused186]": 191, + "[unused187]": 192, + "[unused188]": 193, + "[unused189]": 194, + "[unused190]": 195, + "[unused191]": 196, + "[unused192]": 197, + "[unused193]": 198, + "[unused194]": 199, + "[unused195]": 200, + "[unused196]": 201, + "[unused197]": 202, + "[unused198]": 203, + "[unused199]": 204, + "[unused200]": 205, + "[unused201]": 206, + "[unused202]": 207, + "[unused203]": 208, + "[unused204]": 209, + "[unused205]": 210, + "[unused206]": 211, + "[unused207]": 212, + "[unused208]": 213, + "[unused209]": 214, + "[unused210]": 215, + "[unused211]": 216, + "[unused212]": 217, + "[unused213]": 218, + "[unused214]": 219, + "[unused215]": 220, + "[unused216]": 221, + "[unused217]": 222, + "[unused218]": 223, + "[unused219]": 224, + "[unused220]": 225, + "[unused221]": 226, + "[unused222]": 227, + "[unused223]": 228, + "[unused224]": 229, + "[unused225]": 230, + "[unused226]": 231, + "[unused227]": 232, + "[unused228]": 233, + "[unused229]": 234, + "[unused230]": 235, + "[unused231]": 236, + "[unused232]": 237, + "[unused233]": 238, + "[unused234]": 239, + "[unused235]": 240, + "[unused236]": 241, + "[unused237]": 242, + "[unused238]": 243, + "[unused239]": 244, + "[unused240]": 245, + "[unused241]": 246, + "[unused242]": 247, + "[unused243]": 248, + "[unused244]": 249, + "[unused245]": 250, + "[unused246]": 251, + "[unused247]": 252, + "[unused248]": 253, + "[unused249]": 254, + "[unused250]": 255, + "[unused251]": 256, + "[unused252]": 257, + "[unused253]": 258, + "[unused254]": 259, + "[unused255]": 260, + "[unused256]": 261, + "[unused257]": 262, + "[unused258]": 263, + "[unused259]": 264, + "[unused260]": 265, + "[unused261]": 266, + "[unused262]": 267, + "[unused263]": 268, + "[unused264]": 269, + "[unused265]": 270, + "[unused266]": 271, + "[unused267]": 272, + "[unused268]": 273, + "[unused269]": 274, + "[unused270]": 275, + "[unused271]": 276, + "[unused272]": 277, + "[unused273]": 278, + "[unused274]": 279, + "[unused275]": 280, + "[unused276]": 281, + "[unused277]": 282, + "[unused278]": 283, + "[unused279]": 284, + "[unused280]": 285, + "[unused281]": 286, + "[unused282]": 287, + "[unused283]": 288, + "[unused284]": 289, + "[unused285]": 290, + "[unused286]": 291, + "[unused287]": 292, + "[unused288]": 293, + "[unused289]": 294, + "[unused290]": 295, + "[unused291]": 296, + "[unused292]": 297, + "[unused293]": 298, + "[unused294]": 299, + "[unused295]": 300, + "[unused296]": 301, + "[unused297]": 302, + "[unused298]": 303, + "[unused299]": 304, + "[unused300]": 305, + "[unused301]": 306, + "[unused302]": 307, + "[unused303]": 308, + "[unused304]": 309, + "[unused305]": 310, + "[unused306]": 311, + "[unused307]": 312, + "[unused308]": 313, + "[unused309]": 314, + "[unused310]": 315, + "[unused311]": 316, + "[unused312]": 317, + "[unused313]": 318, + "[unused314]": 319, + "[unused315]": 320, + "[unused316]": 321, + "[unused317]": 322, + "[unused318]": 323, + "[unused319]": 324, + "[unused320]": 325, + "[unused321]": 326, + "[unused322]": 327, + "[unused323]": 328, + "[unused324]": 329, + "[unused325]": 330, + "[unused326]": 331, + "[unused327]": 332, + "[unused328]": 333, + "[unused329]": 334, + "[unused330]": 335, + "[unused331]": 336, + "[unused332]": 337, + "[unused333]": 338, + "[unused334]": 339, + "[unused335]": 340, + "[unused336]": 341, + "[unused337]": 342, + "[unused338]": 343, + "[unused339]": 344, + "[unused340]": 345, + "[unused341]": 346, + "[unused342]": 347, + "[unused343]": 348, + "[unused344]": 349, + "[unused345]": 350, + "[unused346]": 351, + "[unused347]": 352, + "[unused348]": 353, + "[unused349]": 354, + "[unused350]": 355, + "[unused351]": 356, + "[unused352]": 357, + "[unused353]": 358, + "[unused354]": 359, + "[unused355]": 360, + "[unused356]": 361, + "[unused357]": 362, + "[unused358]": 363, + "[unused359]": 364, + "[unused360]": 365, + "[unused361]": 366, + "[unused362]": 367, + "[unused363]": 368, + "[unused364]": 369, + "[unused365]": 370, + "[unused366]": 371, + "[unused367]": 372, + "[unused368]": 373, + "[unused369]": 374, + "[unused370]": 375, + "[unused371]": 376, + "[unused372]": 377, + "[unused373]": 378, + "[unused374]": 379, + "[unused375]": 380, + "[unused376]": 381, + "[unused377]": 382, + "[unused378]": 383, + "[unused379]": 384, + "[unused380]": 385, + "[unused381]": 386, + "[unused382]": 387, + "[unused383]": 388, + "[unused384]": 389, + "[unused385]": 390, + "[unused386]": 391, + "[unused387]": 392, + "[unused388]": 393, + "[unused389]": 394, + "[unused390]": 395, + "[unused391]": 396, + "[unused392]": 397, + "[unused393]": 398, + "[unused394]": 399, + "[unused395]": 400, + "[unused396]": 401, + "[unused397]": 402, + "[unused398]": 403, + "[unused399]": 404, + "[unused400]": 405, + "[unused401]": 406, + "[unused402]": 407, + "[unused403]": 408, + "[unused404]": 409, + "[unused405]": 410, + "[unused406]": 411, + "[unused407]": 412, + "[unused408]": 413, + "[unused409]": 414, + "[unused410]": 415, + "[unused411]": 416, + "[unused412]": 417, + "[unused413]": 418, + "[unused414]": 419, + "[unused415]": 420, + "[unused416]": 421, + "[unused417]": 422, + "[unused418]": 423, + "[unused419]": 424, + "[unused420]": 425, + "[unused421]": 426, + "[unused422]": 427, + "[unused423]": 428, + "[unused424]": 429, + "[unused425]": 430, + "[unused426]": 431, + "[unused427]": 432, + "[unused428]": 433, + "[unused429]": 434, + "[unused430]": 435, + "[unused431]": 436, + "[unused432]": 437, + "[unused433]": 438, + "[unused434]": 439, + "[unused435]": 440, + "[unused436]": 441, + "[unused437]": 442, + "[unused438]": 443, + "[unused439]": 444, + "[unused440]": 445, + "[unused441]": 446, + "[unused442]": 447, + "[unused443]": 448, + "[unused444]": 449, + "[unused445]": 450, + "[unused446]": 451, + "[unused447]": 452, + "[unused448]": 453, + "[unused449]": 454, + "[unused450]": 455, + "[unused451]": 456, + "[unused452]": 457, + "[unused453]": 458, + "[unused454]": 459, + "[unused455]": 460, + "[unused456]": 461, + "[unused457]": 462, + "[unused458]": 463, + "[unused459]": 464, + "[unused460]": 465, + "[unused461]": 466, + "[unused462]": 467, + "[unused463]": 468, + "[unused464]": 469, + "[unused465]": 470, + "[unused466]": 471, + "[unused467]": 472, + "[unused468]": 473, + "[unused469]": 474, + "[unused470]": 475, + "[unused471]": 476, + "[unused472]": 477, + "[unused473]": 478, + "[unused474]": 479, + "[unused475]": 480, + "[unused476]": 481, + "[unused477]": 482, + "[unused478]": 483, + "[unused479]": 484, + "[unused480]": 485, + "[unused481]": 486, + "[unused482]": 487, + "[unused483]": 488, + "[unused484]": 489, + "[unused485]": 490, + "[unused486]": 491, + "[unused487]": 492, + "[unused488]": 493, + "[unused489]": 494, + "[unused490]": 495, + "[unused491]": 496, + "[unused492]": 497, + "[unused493]": 498, + "[unused494]": 499, + "[unused495]": 500, + "[unused496]": 501, + "[unused497]": 502, + "[unused498]": 503, + "[unused499]": 504, + "[unused500]": 505, + "[unused501]": 506, + "[unused502]": 507, + "[unused503]": 508, + "[unused504]": 509, + "[unused505]": 510, + "[unused506]": 511, + "[unused507]": 512, + "[unused508]": 513, + "[unused509]": 514, + "[unused510]": 515, + "[unused511]": 516, + "[unused512]": 517, + "[unused513]": 518, + "[unused514]": 519, + "[unused515]": 520, + "[unused516]": 521, + "[unused517]": 522, + "[unused518]": 523, + "[unused519]": 524, + "[unused520]": 525, + "[unused521]": 526, + "[unused522]": 527, + "[unused523]": 528, + "[unused524]": 529, + "[unused525]": 530, + "[unused526]": 531, + "[unused527]": 532, + "[unused528]": 533, + "[unused529]": 534, + "[unused530]": 535, + "[unused531]": 536, + "[unused532]": 537, + "[unused533]": 538, + "[unused534]": 539, + "[unused535]": 540, + "[unused536]": 541, + "[unused537]": 542, + "[unused538]": 543, + "[unused539]": 544, + "[unused540]": 545, + "[unused541]": 546, + "[unused542]": 547, + "[unused543]": 548, + "[unused544]": 549, + "[unused545]": 550, + "[unused546]": 551, + "[unused547]": 552, + "[unused548]": 553, + "[unused549]": 554, + "[unused550]": 555, + "[unused551]": 556, + "[unused552]": 557, + "[unused553]": 558, + "[unused554]": 559, + "[unused555]": 560, + "[unused556]": 561, + "[unused557]": 562, + "[unused558]": 563, + "[unused559]": 564, + "[unused560]": 565, + "[unused561]": 566, + "[unused562]": 567, + "[unused563]": 568, + "[unused564]": 569, + "[unused565]": 570, + "[unused566]": 571, + "[unused567]": 572, + "[unused568]": 573, + "[unused569]": 574, + "[unused570]": 575, + "[unused571]": 576, + "[unused572]": 577, + "[unused573]": 578, + "[unused574]": 579, + "[unused575]": 580, + "[unused576]": 581, + "[unused577]": 582, + "[unused578]": 583, + "[unused579]": 584, + "[unused580]": 585, + "[unused581]": 586, + "[unused582]": 587, + "[unused583]": 588, + "[unused584]": 589, + "[unused585]": 590, + "[unused586]": 591, + "[unused587]": 592, + "[unused588]": 593, + "[unused589]": 594, + "[unused590]": 595, + "[unused591]": 596, + "[unused592]": 597, + "[unused593]": 598, + "[unused594]": 599, + "[unused595]": 600, + "[unused596]": 601, + "[unused597]": 602, + "[unused598]": 603, + "[unused599]": 604, + "[unused600]": 605, + "[unused601]": 606, + "[unused602]": 607, + "[unused603]": 608, + "[unused604]": 609, + "[unused605]": 610, + "[unused606]": 611, + "[unused607]": 612, + "[unused608]": 613, + "[unused609]": 614, + "[unused610]": 615, + "[unused611]": 616, + "[unused612]": 617, + "[unused613]": 618, + "[unused614]": 619, + "[unused615]": 620, + "[unused616]": 621, + "[unused617]": 622, + "[unused618]": 623, + "[unused619]": 624, + "[unused620]": 625, + "[unused621]": 626, + "[unused622]": 627, + "[unused623]": 628, + "[unused624]": 629, + "[unused625]": 630, + "[unused626]": 631, + "[unused627]": 632, + "[unused628]": 633, + "[unused629]": 634, + "[unused630]": 635, + "[unused631]": 636, + "[unused632]": 637, + "[unused633]": 638, + "[unused634]": 639, + "[unused635]": 640, + "[unused636]": 641, + "[unused637]": 642, + "[unused638]": 643, + "[unused639]": 644, + "[unused640]": 645, + "[unused641]": 646, + "[unused642]": 647, + "[unused643]": 648, + "[unused644]": 649, + "[unused645]": 650, + "[unused646]": 651, + "[unused647]": 652, + "[unused648]": 653, + "[unused649]": 654, + "[unused650]": 655, + "[unused651]": 656, + "[unused652]": 657, + "[unused653]": 658, + "[unused654]": 659, + "[unused655]": 660, + "[unused656]": 661, + "[unused657]": 662, + "[unused658]": 663, + "[unused659]": 664, + "[unused660]": 665, + "[unused661]": 666, + "[unused662]": 667, + "[unused663]": 668, + "[unused664]": 669, + "[unused665]": 670, + "[unused666]": 671, + "[unused667]": 672, + "[unused668]": 673, + "[unused669]": 674, + "[unused670]": 675, + "[unused671]": 676, + "[unused672]": 677, + "[unused673]": 678, + "[unused674]": 679, + "[unused675]": 680, + "[unused676]": 681, + "[unused677]": 682, + "[unused678]": 683, + "[unused679]": 684, + "[unused680]": 685, + "[unused681]": 686, + "[unused682]": 687, + "[unused683]": 688, + "[unused684]": 689, + "[unused685]": 690, + "[unused686]": 691, + "[unused687]": 692, + "[unused688]": 693, + "[unused689]": 694, + "[unused690]": 695, + "[unused691]": 696, + "[unused692]": 697, + "[unused693]": 698, + "[unused694]": 699, + "[unused695]": 700, + "[unused696]": 701, + "[unused697]": 702, + "[unused698]": 703, + "[unused699]": 704, + "[unused700]": 705, + "[unused701]": 706, + "[unused702]": 707, + "[unused703]": 708, + "[unused704]": 709, + "[unused705]": 710, + "[unused706]": 711, + "[unused707]": 712, + "[unused708]": 713, + "[unused709]": 714, + "[unused710]": 715, + "[unused711]": 716, + "[unused712]": 717, + "[unused713]": 718, + "[unused714]": 719, + "[unused715]": 720, + "[unused716]": 721, + "[unused717]": 722, + "[unused718]": 723, + "[unused719]": 724, + "[unused720]": 725, + "[unused721]": 726, + "[unused722]": 727, + "[unused723]": 728, + "[unused724]": 729, + "[unused725]": 730, + "[unused726]": 731, + "[unused727]": 732, + "[unused728]": 733, + "[unused729]": 734, + "[unused730]": 735, + "[unused731]": 736, + "[unused732]": 737, + "[unused733]": 738, + "[unused734]": 739, + "[unused735]": 740, + "[unused736]": 741, + "[unused737]": 742, + "[unused738]": 743, + "[unused739]": 744, + "[unused740]": 745, + "[unused741]": 746, + "[unused742]": 747, + "[unused743]": 748, + "[unused744]": 749, + "[unused745]": 750, + "[unused746]": 751, + "[unused747]": 752, + "[unused748]": 753, + "[unused749]": 754, + "[unused750]": 755, + "[unused751]": 756, + "[unused752]": 757, + "[unused753]": 758, + "[unused754]": 759, + "[unused755]": 760, + "[unused756]": 761, + "[unused757]": 762, + "[unused758]": 763, + "[unused759]": 764, + "[unused760]": 765, + "[unused761]": 766, + "[unused762]": 767, + "[unused763]": 768, + "[unused764]": 769, + "[unused765]": 770, + "[unused766]": 771, + "[unused767]": 772, + "[unused768]": 773, + "[unused769]": 774, + "[unused770]": 775, + "[unused771]": 776, + "[unused772]": 777, + "[unused773]": 778, + "[unused774]": 779, + "[unused775]": 780, + "[unused776]": 781, + "[unused777]": 782, + "[unused778]": 783, + "[unused779]": 784, + "[unused780]": 785, + "[unused781]": 786, + "[unused782]": 787, + "[unused783]": 788, + "[unused784]": 789, + "[unused785]": 790, + "[unused786]": 791, + "[unused787]": 792, + "[unused788]": 793, + "[unused789]": 794, + "[unused790]": 795, + "[unused791]": 796, + "[unused792]": 797, + "[unused793]": 798, + "[unused794]": 799, + "[unused795]": 800, + "[unused796]": 801, + "[unused797]": 802, + "[unused798]": 803, + "[unused799]": 804, + "[unused800]": 805, + "[unused801]": 806, + "[unused802]": 807, + "[unused803]": 808, + "[unused804]": 809, + "[unused805]": 810, + "[unused806]": 811, + "[unused807]": 812, + "[unused808]": 813, + "[unused809]": 814, + "[unused810]": 815, + "[unused811]": 816, + "[unused812]": 817, + "[unused813]": 818, + "[unused814]": 819, + "[unused815]": 820, + "[unused816]": 821, + "[unused817]": 822, + "[unused818]": 823, + "[unused819]": 824, + "[unused820]": 825, + "[unused821]": 826, + "[unused822]": 827, + "[unused823]": 828, + "[unused824]": 829, + "[unused825]": 830, + "[unused826]": 831, + "[unused827]": 832, + "[unused828]": 833, + "[unused829]": 834, + "[unused830]": 835, + "[unused831]": 836, + "[unused832]": 837, + "[unused833]": 838, + "[unused834]": 839, + "[unused835]": 840, + "[unused836]": 841, + "[unused837]": 842, + "[unused838]": 843, + "[unused839]": 844, + "[unused840]": 845, + "[unused841]": 846, + "[unused842]": 847, + "[unused843]": 848, + "[unused844]": 849, + "[unused845]": 850, + "[unused846]": 851, + "[unused847]": 852, + "[unused848]": 853, + "[unused849]": 854, + "[unused850]": 855, + "[unused851]": 856, + "[unused852]": 857, + "[unused853]": 858, + "[unused854]": 859, + "[unused855]": 860, + "[unused856]": 861, + "[unused857]": 862, + "[unused858]": 863, + "[unused859]": 864, + "[unused860]": 865, + "[unused861]": 866, + "[unused862]": 867, + "[unused863]": 868, + "[unused864]": 869, + "[unused865]": 870, + "[unused866]": 871, + "[unused867]": 872, + "[unused868]": 873, + "[unused869]": 874, + "[unused870]": 875, + "[unused871]": 876, + "[unused872]": 877, + "[unused873]": 878, + "[unused874]": 879, + "[unused875]": 880, + "[unused876]": 881, + "[unused877]": 882, + "[unused878]": 883, + "[unused879]": 884, + "[unused880]": 885, + "[unused881]": 886, + "[unused882]": 887, + "[unused883]": 888, + "[unused884]": 889, + "[unused885]": 890, + "[unused886]": 891, + "[unused887]": 892, + "[unused888]": 893, + "[unused889]": 894, + "[unused890]": 895, + "[unused891]": 896, + "[unused892]": 897, + "[unused893]": 898, + "[unused894]": 899, + "[unused895]": 900, + "[unused896]": 901, + "[unused897]": 902, + "[unused898]": 903, + "[unused899]": 904, + "[unused900]": 905, + "[unused901]": 906, + "[unused902]": 907, + "[unused903]": 908, + "[unused904]": 909, + "[unused905]": 910, + "[unused906]": 911, + "[unused907]": 912, + "[unused908]": 913, + "[unused909]": 914, + "[unused910]": 915, + "[unused911]": 916, + "[unused912]": 917, + "[unused913]": 918, + "[unused914]": 919, + "[unused915]": 920, + "[unused916]": 921, + "[unused917]": 922, + "[unused918]": 923, + "[unused919]": 924, + "[unused920]": 925, + "[unused921]": 926, + "[unused922]": 927, + "[unused923]": 928, + "[unused924]": 929, + "[unused925]": 930, + "[unused926]": 931, + "[unused927]": 932, + "[unused928]": 933, + "[unused929]": 934, + "[unused930]": 935, + "[unused931]": 936, + "[unused932]": 937, + "[unused933]": 938, + "[unused934]": 939, + "[unused935]": 940, + "[unused936]": 941, + "[unused937]": 942, + "[unused938]": 943, + "[unused939]": 944, + "[unused940]": 945, + "[unused941]": 946, + "[unused942]": 947, + "[unused943]": 948, + "[unused944]": 949, + "[unused945]": 950, + "[unused946]": 951, + "[unused947]": 952, + "[unused948]": 953, + "[unused949]": 954, + "[unused950]": 955, + "[unused951]": 956, + "[unused952]": 957, + "[unused953]": 958, + "[unused954]": 959, + "[unused955]": 960, + "[unused956]": 961, + "[unused957]": 962, + "[unused958]": 963, + "[unused959]": 964, + "[unused960]": 965, + "[unused961]": 966, + "[unused962]": 967, + "[unused963]": 968, + "[unused964]": 969, + "[unused965]": 970, + "[unused966]": 971, + "[unused967]": 972, + "[unused968]": 973, + "[unused969]": 974, + "[unused970]": 975, + "[unused971]": 976, + "[unused972]": 977, + "[unused973]": 978, + "[unused974]": 979, + "[unused975]": 980, + "[unused976]": 981, + "[unused977]": 982, + "[unused978]": 983, + "[unused979]": 984, + "[unused980]": 985, + "[unused981]": 986, + "[unused982]": 987, + "[unused983]": 988, + "[unused984]": 989, + "[unused985]": 990, + "[unused986]": 991, + "[unused987]": 992, + "[unused988]": 993, + "[unused989]": 994, + "[unused990]": 995, + "[unused991]": 996, + "[unused992]": 997, + "[unused993]": 998, + "!": 999, + "\"": 1000, + "#": 1001, + "$": 1002, + "%": 1003, + "&": 1004, + "'": 1005, + "(": 1006, + ")": 1007, + "*": 1008, + "+": 1009, + ",": 1010, + "-": 1011, + ".": 1012, + "/": 1013, + "0": 1014, + "1": 1015, + "2": 1016, + "3": 1017, + "4": 1018, + "5": 1019, + "6": 1020, + "7": 1021, + "8": 1022, + "9": 1023, + ":": 1024, + ";": 1025, + "<": 1026, + "=": 1027, + ">": 1028, + "?": 1029, + "@": 1030, + "[": 1031, + "\\": 1032, + "]": 1033, + "^": 1034, + "_": 1035, + "`": 1036, + "a": 1037, + "b": 1038, + "c": 1039, + "d": 1040, + "e": 1041, + "f": 1042, + "g": 1043, + "h": 1044, + "i": 1045, + "j": 1046, + "k": 1047, + "l": 1048, + "m": 1049, + "n": 1050, + "o": 1051, + "p": 1052, + "q": 1053, + "r": 1054, + "s": 1055, + "t": 1056, + "u": 1057, + "v": 1058, + "w": 1059, + "x": 1060, + "y": 1061, + "z": 1062, + "{": 1063, + "|": 1064, + "}": 1065, + "~": 1066, + "¡": 1067, + "¢": 1068, + "£": 1069, + "¤": 1070, + "¥": 1071, + "¦": 1072, + "§": 1073, + "¨": 1074, + "©": 1075, + "ª": 1076, + "«": 1077, + "¬": 1078, + "®": 1079, + "°": 1080, + "±": 1081, + "²": 1082, + "³": 1083, + "´": 1084, + "µ": 1085, + "¶": 1086, + "·": 1087, + "¹": 1088, + "º": 1089, + "»": 1090, + "¼": 1091, + "½": 1092, + "¾": 1093, + "¿": 1094, + "×": 1095, + "ß": 1096, + "æ": 1097, + "ð": 1098, + "÷": 1099, + "ø": 1100, + "þ": 1101, + "đ": 1102, + "ħ": 1103, + "ı": 1104, + "ł": 1105, + "ŋ": 1106, + "œ": 1107, + "ƒ": 1108, + "ɐ": 1109, + "ɑ": 1110, + "ɒ": 1111, + "ɔ": 1112, + "ɕ": 1113, + "ə": 1114, + "ɛ": 1115, + "ɡ": 1116, + "ɣ": 1117, + "ɨ": 1118, + "ɪ": 1119, + "ɫ": 1120, + "ɬ": 1121, + "ɯ": 1122, + "ɲ": 1123, + "ɴ": 1124, + "ɹ": 1125, + "ɾ": 1126, + "ʀ": 1127, + "ʁ": 1128, + "ʂ": 1129, + "ʃ": 1130, + "ʉ": 1131, + "ʊ": 1132, + "ʋ": 1133, + "ʌ": 1134, + "ʎ": 1135, + "ʐ": 1136, + "ʑ": 1137, + "ʒ": 1138, + "ʔ": 1139, + "ʰ": 1140, + "ʲ": 1141, + "ʳ": 1142, + "ʷ": 1143, + "ʸ": 1144, + "ʻ": 1145, + "ʼ": 1146, + "ʾ": 1147, + "ʿ": 1148, + "ˈ": 1149, + "ː": 1150, + "ˡ": 1151, + "ˢ": 1152, + "ˣ": 1153, + "ˤ": 1154, + "α": 1155, + "β": 1156, + "γ": 1157, + "δ": 1158, + "ε": 1159, + "ζ": 1160, + "η": 1161, + "θ": 1162, + "ι": 1163, + "κ": 1164, + "λ": 1165, + "μ": 1166, + "ν": 1167, + "ξ": 1168, + "ο": 1169, + "π": 1170, + "ρ": 1171, + "ς": 1172, + "σ": 1173, + "τ": 1174, + "υ": 1175, + "φ": 1176, + "χ": 1177, + "ψ": 1178, + "ω": 1179, + "а": 1180, + "б": 1181, + "в": 1182, + "г": 1183, + "д": 1184, + "е": 1185, + "ж": 1186, + "з": 1187, + "и": 1188, + "к": 1189, + "л": 1190, + "м": 1191, + "н": 1192, + "о": 1193, + "п": 1194, + "р": 1195, + "с": 1196, + "т": 1197, + "у": 1198, + "ф": 1199, + "х": 1200, + "ц": 1201, + "ч": 1202, + "ш": 1203, + "щ": 1204, + "ъ": 1205, + "ы": 1206, + "ь": 1207, + "э": 1208, + "ю": 1209, + "я": 1210, + "ђ": 1211, + "є": 1212, + "і": 1213, + "ј": 1214, + "љ": 1215, + "њ": 1216, + "ћ": 1217, + "ӏ": 1218, + "ա": 1219, + "բ": 1220, + "գ": 1221, + "դ": 1222, + "ե": 1223, + "թ": 1224, + "ի": 1225, + "լ": 1226, + "կ": 1227, + "հ": 1228, + "մ": 1229, + "յ": 1230, + "ն": 1231, + "ո": 1232, + "պ": 1233, + "ս": 1234, + "վ": 1235, + "տ": 1236, + "ր": 1237, + "ւ": 1238, + "ք": 1239, + "־": 1240, + "א": 1241, + "ב": 1242, + "ג": 1243, + "ד": 1244, + "ה": 1245, + "ו": 1246, + "ז": 1247, + "ח": 1248, + "ט": 1249, + "י": 1250, + "ך": 1251, + "כ": 1252, + "ל": 1253, + "ם": 1254, + "מ": 1255, + "ן": 1256, + "נ": 1257, + "ס": 1258, + "ע": 1259, + "ף": 1260, + "פ": 1261, + "ץ": 1262, + "צ": 1263, + "ק": 1264, + "ר": 1265, + "ש": 1266, + "ת": 1267, + "،": 1268, + "ء": 1269, + "ا": 1270, + "ب": 1271, + "ة": 1272, + "ت": 1273, + "ث": 1274, + "ج": 1275, + "ح": 1276, + "خ": 1277, + "د": 1278, + "ذ": 1279, + "ر": 1280, + "ز": 1281, + "س": 1282, + "ش": 1283, + "ص": 1284, + "ض": 1285, + "ط": 1286, + "ظ": 1287, + "ع": 1288, + "غ": 1289, + "ـ": 1290, + "ف": 1291, + "ق": 1292, + "ك": 1293, + "ل": 1294, + "م": 1295, + "ن": 1296, + "ه": 1297, + "و": 1298, + "ى": 1299, + "ي": 1300, + "ٹ": 1301, + "پ": 1302, + "چ": 1303, + "ک": 1304, + "گ": 1305, + "ں": 1306, + "ھ": 1307, + "ہ": 1308, + "ی": 1309, + "ے": 1310, + "अ": 1311, + "आ": 1312, + "उ": 1313, + "ए": 1314, + "क": 1315, + "ख": 1316, + "ग": 1317, + "च": 1318, + "ज": 1319, + "ट": 1320, + "ड": 1321, + "ण": 1322, + "त": 1323, + "थ": 1324, + "द": 1325, + "ध": 1326, + "न": 1327, + "प": 1328, + "ब": 1329, + "भ": 1330, + "म": 1331, + "य": 1332, + "र": 1333, + "ल": 1334, + "व": 1335, + "श": 1336, + "ष": 1337, + "स": 1338, + "ह": 1339, + "ा": 1340, + "ि": 1341, + "ी": 1342, + "ो": 1343, + "।": 1344, + "॥": 1345, + "ং": 1346, + "অ": 1347, + "আ": 1348, + "ই": 1349, + "উ": 1350, + "এ": 1351, + "ও": 1352, + "ক": 1353, + "খ": 1354, + "গ": 1355, + "চ": 1356, + "ছ": 1357, + "জ": 1358, + "ট": 1359, + "ড": 1360, + "ণ": 1361, + "ত": 1362, + "থ": 1363, + "দ": 1364, + "ধ": 1365, + "ন": 1366, + "প": 1367, + "ব": 1368, + "ভ": 1369, + "ম": 1370, + "য": 1371, + "র": 1372, + "ল": 1373, + "শ": 1374, + "ষ": 1375, + "স": 1376, + "হ": 1377, + "া": 1378, + "ি": 1379, + "ী": 1380, + "ে": 1381, + "க": 1382, + "ச": 1383, + "ட": 1384, + "த": 1385, + "ந": 1386, + "ன": 1387, + "ப": 1388, + "ம": 1389, + "ய": 1390, + "ர": 1391, + "ல": 1392, + "ள": 1393, + "வ": 1394, + "ா": 1395, + "ி": 1396, + "ு": 1397, + "ே": 1398, + "ை": 1399, + "ನ": 1400, + "ರ": 1401, + "ಾ": 1402, + "ක": 1403, + "ය": 1404, + "ර": 1405, + "ල": 1406, + "ව": 1407, + "ා": 1408, + "ก": 1409, + "ง": 1410, + "ต": 1411, + "ท": 1412, + "น": 1413, + "พ": 1414, + "ม": 1415, + "ย": 1416, + "ร": 1417, + "ล": 1418, + "ว": 1419, + "ส": 1420, + "อ": 1421, + "า": 1422, + "เ": 1423, + "་": 1424, + "།": 1425, + "ག": 1426, + "ང": 1427, + "ད": 1428, + "ན": 1429, + "པ": 1430, + "བ": 1431, + "མ": 1432, + "འ": 1433, + "ར": 1434, + "ལ": 1435, + "ས": 1436, + "မ": 1437, + "ა": 1438, + "ბ": 1439, + "გ": 1440, + "დ": 1441, + "ე": 1442, + "ვ": 1443, + "თ": 1444, + "ი": 1445, + "კ": 1446, + "ლ": 1447, + "მ": 1448, + "ნ": 1449, + "ო": 1450, + "რ": 1451, + "ს": 1452, + "ტ": 1453, + "უ": 1454, + "ᄀ": 1455, + "ᄂ": 1456, + "ᄃ": 1457, + "ᄅ": 1458, + "ᄆ": 1459, + "ᄇ": 1460, + "ᄉ": 1461, + "ᄊ": 1462, + "ᄋ": 1463, + "ᄌ": 1464, + "ᄎ": 1465, + "ᄏ": 1466, + "ᄐ": 1467, + "ᄑ": 1468, + "ᄒ": 1469, + "ᅡ": 1470, + "ᅢ": 1471, + "ᅥ": 1472, + "ᅦ": 1473, + "ᅧ": 1474, + "ᅩ": 1475, + "ᅪ": 1476, + "ᅭ": 1477, + "ᅮ": 1478, + "ᅯ": 1479, + "ᅲ": 1480, + "ᅳ": 1481, + "ᅴ": 1482, + "ᅵ": 1483, + "ᆨ": 1484, + "ᆫ": 1485, + "ᆯ": 1486, + "ᆷ": 1487, + "ᆸ": 1488, + "ᆼ": 1489, + "ᴬ": 1490, + "ᴮ": 1491, + "ᴰ": 1492, + "ᴵ": 1493, + "ᴺ": 1494, + "ᵀ": 1495, + "ᵃ": 1496, + "ᵇ": 1497, + "ᵈ": 1498, + "ᵉ": 1499, + "ᵍ": 1500, + "ᵏ": 1501, + "ᵐ": 1502, + "ᵒ": 1503, + "ᵖ": 1504, + "ᵗ": 1505, + "ᵘ": 1506, + "ᵢ": 1507, + "ᵣ": 1508, + "ᵤ": 1509, + "ᵥ": 1510, + "ᶜ": 1511, + "ᶠ": 1512, + "‐": 1513, + "‑": 1514, + "‒": 1515, + "–": 1516, + "—": 1517, + "―": 1518, + "‖": 1519, + "‘": 1520, + "’": 1521, + "‚": 1522, + "“": 1523, + "”": 1524, + "„": 1525, + "†": 1526, + "‡": 1527, + "•": 1528, + "…": 1529, + "‰": 1530, + "′": 1531, + "″": 1532, + "›": 1533, + "‿": 1534, + "⁄": 1535, + "⁰": 1536, + "ⁱ": 1537, + "⁴": 1538, + "⁵": 1539, + "⁶": 1540, + "⁷": 1541, + "⁸": 1542, + "⁹": 1543, + "⁺": 1544, + "⁻": 1545, + "ⁿ": 1546, + "₀": 1547, + "₁": 1548, + "₂": 1549, + "₃": 1550, + "₄": 1551, + "₅": 1552, + "₆": 1553, + "₇": 1554, + "₈": 1555, + "₉": 1556, + "₊": 1557, + "₍": 1558, + "₎": 1559, + "ₐ": 1560, + "ₑ": 1561, + "ₒ": 1562, + "ₓ": 1563, + "ₕ": 1564, + "ₖ": 1565, + "ₗ": 1566, + "ₘ": 1567, + "ₙ": 1568, + "ₚ": 1569, + "ₛ": 1570, + "ₜ": 1571, + "₤": 1572, + "₩": 1573, + "€": 1574, + "₱": 1575, + "₹": 1576, + "ℓ": 1577, + "№": 1578, + "ℝ": 1579, + "™": 1580, + "⅓": 1581, + "⅔": 1582, + "←": 1583, + "↑": 1584, + "→": 1585, + "↓": 1586, + "↔": 1587, + "↦": 1588, + "⇄": 1589, + "⇌": 1590, + "⇒": 1591, + "∂": 1592, + "∅": 1593, + "∆": 1594, + "∇": 1595, + "∈": 1596, + "−": 1597, + "∗": 1598, + "∘": 1599, + "√": 1600, + "∞": 1601, + "∧": 1602, + "∨": 1603, + "∩": 1604, + "∪": 1605, + "≈": 1606, + "≡": 1607, + "≤": 1608, + "≥": 1609, + "⊂": 1610, + "⊆": 1611, + "⊕": 1612, + "⊗": 1613, + "⋅": 1614, + "─": 1615, + "│": 1616, + "■": 1617, + "▪": 1618, + "●": 1619, + "★": 1620, + "☆": 1621, + "☉": 1622, + "♠": 1623, + "♣": 1624, + "♥": 1625, + "♦": 1626, + "♭": 1627, + "♯": 1628, + "⟨": 1629, + "⟩": 1630, + "ⱼ": 1631, + "⺩": 1632, + "⺼": 1633, + "⽥": 1634, + "、": 1635, + "。": 1636, + "〈": 1637, + "〉": 1638, + "《": 1639, + "》": 1640, + "「": 1641, + "」": 1642, + "『": 1643, + "』": 1644, + "〜": 1645, + "あ": 1646, + "い": 1647, + "う": 1648, + "え": 1649, + "お": 1650, + "か": 1651, + "き": 1652, + "く": 1653, + "け": 1654, + "こ": 1655, + "さ": 1656, + "し": 1657, + "す": 1658, + "せ": 1659, + "そ": 1660, + "た": 1661, + "ち": 1662, + "っ": 1663, + "つ": 1664, + "て": 1665, + "と": 1666, + "な": 1667, + "に": 1668, + "ぬ": 1669, + "ね": 1670, + "の": 1671, + "は": 1672, + "ひ": 1673, + "ふ": 1674, + "へ": 1675, + "ほ": 1676, + "ま": 1677, + "み": 1678, + "む": 1679, + "め": 1680, + "も": 1681, + "や": 1682, + "ゆ": 1683, + "よ": 1684, + "ら": 1685, + "り": 1686, + "る": 1687, + "れ": 1688, + "ろ": 1689, + "を": 1690, + "ん": 1691, + "ァ": 1692, + "ア": 1693, + "ィ": 1694, + "イ": 1695, + "ウ": 1696, + "ェ": 1697, + "エ": 1698, + "オ": 1699, + "カ": 1700, + "キ": 1701, + "ク": 1702, + "ケ": 1703, + "コ": 1704, + "サ": 1705, + "シ": 1706, + "ス": 1707, + "セ": 1708, + "タ": 1709, + "チ": 1710, + "ッ": 1711, + "ツ": 1712, + "テ": 1713, + "ト": 1714, + "ナ": 1715, + "ニ": 1716, + "ノ": 1717, + "ハ": 1718, + "ヒ": 1719, + "フ": 1720, + "ヘ": 1721, + "ホ": 1722, + "マ": 1723, + "ミ": 1724, + "ム": 1725, + "メ": 1726, + "モ": 1727, + "ャ": 1728, + "ュ": 1729, + "ョ": 1730, + "ラ": 1731, + "リ": 1732, + "ル": 1733, + "レ": 1734, + "ロ": 1735, + "ワ": 1736, + "ン": 1737, + "・": 1738, + "ー": 1739, + "一": 1740, + "三": 1741, + "上": 1742, + "下": 1743, + "不": 1744, + "世": 1745, + "中": 1746, + "主": 1747, + "久": 1748, + "之": 1749, + "也": 1750, + "事": 1751, + "二": 1752, + "五": 1753, + "井": 1754, + "京": 1755, + "人": 1756, + "亻": 1757, + "仁": 1758, + "介": 1759, + "代": 1760, + "仮": 1761, + "伊": 1762, + "会": 1763, + "佐": 1764, + "侍": 1765, + "保": 1766, + "信": 1767, + "健": 1768, + "元": 1769, + "光": 1770, + "八": 1771, + "公": 1772, + "内": 1773, + "出": 1774, + "分": 1775, + "前": 1776, + "劉": 1777, + "力": 1778, + "加": 1779, + "勝": 1780, + "北": 1781, + "区": 1782, + "十": 1783, + "千": 1784, + "南": 1785, + "博": 1786, + "原": 1787, + "口": 1788, + "古": 1789, + "史": 1790, + "司": 1791, + "合": 1792, + "吉": 1793, + "同": 1794, + "名": 1795, + "和": 1796, + "囗": 1797, + "四": 1798, + "国": 1799, + "國": 1800, + "土": 1801, + "地": 1802, + "坂": 1803, + "城": 1804, + "堂": 1805, + "場": 1806, + "士": 1807, + "夏": 1808, + "外": 1809, + "大": 1810, + "天": 1811, + "太": 1812, + "夫": 1813, + "奈": 1814, + "女": 1815, + "子": 1816, + "学": 1817, + "宀": 1818, + "宇": 1819, + "安": 1820, + "宗": 1821, + "定": 1822, + "宣": 1823, + "宮": 1824, + "家": 1825, + "宿": 1826, + "寺": 1827, + "將": 1828, + "小": 1829, + "尚": 1830, + "山": 1831, + "岡": 1832, + "島": 1833, + "崎": 1834, + "川": 1835, + "州": 1836, + "巿": 1837, + "帝": 1838, + "平": 1839, + "年": 1840, + "幸": 1841, + "广": 1842, + "弘": 1843, + "張": 1844, + "彳": 1845, + "後": 1846, + "御": 1847, + "德": 1848, + "心": 1849, + "忄": 1850, + "志": 1851, + "忠": 1852, + "愛": 1853, + "成": 1854, + "我": 1855, + "戦": 1856, + "戸": 1857, + "手": 1858, + "扌": 1859, + "政": 1860, + "文": 1861, + "新": 1862, + "方": 1863, + "日": 1864, + "明": 1865, + "星": 1866, + "春": 1867, + "昭": 1868, + "智": 1869, + "曲": 1870, + "書": 1871, + "月": 1872, + "有": 1873, + "朝": 1874, + "木": 1875, + "本": 1876, + "李": 1877, + "村": 1878, + "東": 1879, + "松": 1880, + "林": 1881, + "森": 1882, + "楊": 1883, + "樹": 1884, + "橋": 1885, + "歌": 1886, + "止": 1887, + "正": 1888, + "武": 1889, + "比": 1890, + "氏": 1891, + "民": 1892, + "水": 1893, + "氵": 1894, + "氷": 1895, + "永": 1896, + "江": 1897, + "沢": 1898, + "河": 1899, + "治": 1900, + "法": 1901, + "海": 1902, + "清": 1903, + "漢": 1904, + "瀬": 1905, + "火": 1906, + "版": 1907, + "犬": 1908, + "王": 1909, + "生": 1910, + "田": 1911, + "男": 1912, + "疒": 1913, + "発": 1914, + "白": 1915, + "的": 1916, + "皇": 1917, + "目": 1918, + "相": 1919, + "省": 1920, + "真": 1921, + "石": 1922, + "示": 1923, + "社": 1924, + "神": 1925, + "福": 1926, + "禾": 1927, + "秀": 1928, + "秋": 1929, + "空": 1930, + "立": 1931, + "章": 1932, + "竹": 1933, + "糹": 1934, + "美": 1935, + "義": 1936, + "耳": 1937, + "良": 1938, + "艹": 1939, + "花": 1940, + "英": 1941, + "華": 1942, + "葉": 1943, + "藤": 1944, + "行": 1945, + "街": 1946, + "西": 1947, + "見": 1948, + "訁": 1949, + "語": 1950, + "谷": 1951, + "貝": 1952, + "貴": 1953, + "車": 1954, + "軍": 1955, + "辶": 1956, + "道": 1957, + "郎": 1958, + "郡": 1959, + "部": 1960, + "都": 1961, + "里": 1962, + "野": 1963, + "金": 1964, + "鈴": 1965, + "镇": 1966, + "長": 1967, + "門": 1968, + "間": 1969, + "阝": 1970, + "阿": 1971, + "陳": 1972, + "陽": 1973, + "雄": 1974, + "青": 1975, + "面": 1976, + "風": 1977, + "食": 1978, + "香": 1979, + "馬": 1980, + "高": 1981, + "龍": 1982, + "龸": 1983, + "fi": 1984, + "fl": 1985, + "！": 1986, + "（": 1987, + "）": 1988, + "，": 1989, + "－": 1990, + "．": 1991, + "／": 1992, + "：": 1993, + "？": 1994, + "～": 1995, + "the": 1996, + "of": 1997, + "and": 1998, + "in": 1999, + "to": 2000, + "was": 2001, + "he": 2002, + "is": 2003, + "as": 2004, + "for": 2005, + "on": 2006, + "with": 2007, + "that": 2008, + "it": 2009, + "his": 2010, + "by": 2011, + "at": 2012, + "from": 2013, + "her": 2014, + "##s": 2015, + "she": 2016, + "you": 2017, + "had": 2018, + "an": 2019, + "were": 2020, + "but": 2021, + "be": 2022, + "this": 2023, + "are": 2024, + "not": 2025, + "my": 2026, + "they": 2027, + "one": 2028, + "which": 2029, + "or": 2030, + "have": 2031, + "him": 2032, + "me": 2033, + "first": 2034, + "all": 2035, + "also": 2036, + "their": 2037, + "has": 2038, + "up": 2039, + "who": 2040, + "out": 2041, + "been": 2042, + "when": 2043, + "after": 2044, + "there": 2045, + "into": 2046, + "new": 2047, + "two": 2048, + "its": 2049, + "##a": 2050, + "time": 2051, + "would": 2052, + "no": 2053, + "what": 2054, + "about": 2055, + "said": 2056, + "we": 2057, + "over": 2058, + "then": 2059, + "other": 2060, + "so": 2061, + "more": 2062, + "##e": 2063, + "can": 2064, + "if": 2065, + "like": 2066, + "back": 2067, + "them": 2068, + "only": 2069, + "some": 2070, + "could": 2071, + "##i": 2072, + "where": 2073, + "just": 2074, + "##ing": 2075, + "during": 2076, + "before": 2077, + "##n": 2078, + "do": 2079, + "##o": 2080, + "made": 2081, + "school": 2082, + "through": 2083, + "than": 2084, + "now": 2085, + "years": 2086, + "most": 2087, + "world": 2088, + "may": 2089, + "between": 2090, + "down": 2091, + "well": 2092, + "three": 2093, + "##d": 2094, + "year": 2095, + "while": 2096, + "will": 2097, + "##ed": 2098, + "##r": 2099, + "##y": 2100, + "later": 2101, + "##t": 2102, + "city": 2103, + "under": 2104, + "around": 2105, + "did": 2106, + "such": 2107, + "being": 2108, + "used": 2109, + "state": 2110, + "people": 2111, + "part": 2112, + "know": 2113, + "against": 2114, + "your": 2115, + "many": 2116, + "second": 2117, + "university": 2118, + "both": 2119, + "national": 2120, + "##er": 2121, + "these": 2122, + "don": 2123, + "known": 2124, + "off": 2125, + "way": 2126, + "until": 2127, + "re": 2128, + "how": 2129, + "even": 2130, + "get": 2131, + "head": 2132, + "...": 2133, + "didn": 2134, + "##ly": 2135, + "team": 2136, + "american": 2137, + "because": 2138, + "de": 2139, + "##l": 2140, + "born": 2141, + "united": 2142, + "film": 2143, + "since": 2144, + "still": 2145, + "long": 2146, + "work": 2147, + "south": 2148, + "us": 2149, + "became": 2150, + "any": 2151, + "high": 2152, + "again": 2153, + "day": 2154, + "family": 2155, + "see": 2156, + "right": 2157, + "man": 2158, + "eyes": 2159, + "house": 2160, + "season": 2161, + "war": 2162, + "states": 2163, + "including": 2164, + "took": 2165, + "life": 2166, + "north": 2167, + "same": 2168, + "each": 2169, + "called": 2170, + "name": 2171, + "much": 2172, + "place": 2173, + "however": 2174, + "go": 2175, + "four": 2176, + "group": 2177, + "another": 2178, + "found": 2179, + "won": 2180, + "area": 2181, + "here": 2182, + "going": 2183, + "10": 2184, + "away": 2185, + "series": 2186, + "left": 2187, + "home": 2188, + "music": 2189, + "best": 2190, + "make": 2191, + "hand": 2192, + "number": 2193, + "company": 2194, + "several": 2195, + "never": 2196, + "last": 2197, + "john": 2198, + "000": 2199, + "very": 2200, + "album": 2201, + "take": 2202, + "end": 2203, + "good": 2204, + "too": 2205, + "following": 2206, + "released": 2207, + "game": 2208, + "played": 2209, + "little": 2210, + "began": 2211, + "district": 2212, + "##m": 2213, + "old": 2214, + "want": 2215, + "those": 2216, + "side": 2217, + "held": 2218, + "own": 2219, + "early": 2220, + "county": 2221, + "ll": 2222, + "league": 2223, + "use": 2224, + "west": 2225, + "##u": 2226, + "face": 2227, + "think": 2228, + "##es": 2229, + "2010": 2230, + "government": 2231, + "##h": 2232, + "march": 2233, + "came": 2234, + "small": 2235, + "general": 2236, + "town": 2237, + "june": 2238, + "##on": 2239, + "line": 2240, + "based": 2241, + "something": 2242, + "##k": 2243, + "september": 2244, + "thought": 2245, + "looked": 2246, + "along": 2247, + "international": 2248, + "2011": 2249, + "air": 2250, + "july": 2251, + "club": 2252, + "went": 2253, + "january": 2254, + "october": 2255, + "our": 2256, + "august": 2257, + "april": 2258, + "york": 2259, + "12": 2260, + "few": 2261, + "2012": 2262, + "2008": 2263, + "east": 2264, + "show": 2265, + "member": 2266, + "college": 2267, + "2009": 2268, + "father": 2269, + "public": 2270, + "##us": 2271, + "come": 2272, + "men": 2273, + "five": 2274, + "set": 2275, + "station": 2276, + "church": 2277, + "##c": 2278, + "next": 2279, + "former": 2280, + "november": 2281, + "room": 2282, + "party": 2283, + "located": 2284, + "december": 2285, + "2013": 2286, + "age": 2287, + "got": 2288, + "2007": 2289, + "##g": 2290, + "system": 2291, + "let": 2292, + "love": 2293, + "2006": 2294, + "though": 2295, + "every": 2296, + "2014": 2297, + "look": 2298, + "song": 2299, + "water": 2300, + "century": 2301, + "without": 2302, + "body": 2303, + "black": 2304, + "night": 2305, + "within": 2306, + "great": 2307, + "women": 2308, + "single": 2309, + "ve": 2310, + "building": 2311, + "large": 2312, + "population": 2313, + "river": 2314, + "named": 2315, + "band": 2316, + "white": 2317, + "started": 2318, + "##an": 2319, + "once": 2320, + "15": 2321, + "20": 2322, + "should": 2323, + "18": 2324, + "2015": 2325, + "service": 2326, + "top": 2327, + "built": 2328, + "british": 2329, + "open": 2330, + "death": 2331, + "king": 2332, + "moved": 2333, + "local": 2334, + "times": 2335, + "children": 2336, + "february": 2337, + "book": 2338, + "why": 2339, + "11": 2340, + "door": 2341, + "need": 2342, + "president": 2343, + "order": 2344, + "final": 2345, + "road": 2346, + "wasn": 2347, + "although": 2348, + "due": 2349, + "major": 2350, + "died": 2351, + "village": 2352, + "third": 2353, + "knew": 2354, + "2016": 2355, + "asked": 2356, + "turned": 2357, + "st": 2358, + "wanted": 2359, + "say": 2360, + "##p": 2361, + "together": 2362, + "received": 2363, + "main": 2364, + "son": 2365, + "served": 2366, + "different": 2367, + "##en": 2368, + "behind": 2369, + "himself": 2370, + "felt": 2371, + "members": 2372, + "power": 2373, + "football": 2374, + "law": 2375, + "voice": 2376, + "play": 2377, + "##in": 2378, + "near": 2379, + "park": 2380, + "history": 2381, + "30": 2382, + "having": 2383, + "2005": 2384, + "16": 2385, + "##man": 2386, + "saw": 2387, + "mother": 2388, + "##al": 2389, + "army": 2390, + "point": 2391, + "front": 2392, + "help": 2393, + "english": 2394, + "street": 2395, + "art": 2396, + "late": 2397, + "hands": 2398, + "games": 2399, + "award": 2400, + "##ia": 2401, + "young": 2402, + "14": 2403, + "put": 2404, + "published": 2405, + "country": 2406, + "division": 2407, + "across": 2408, + "told": 2409, + "13": 2410, + "often": 2411, + "ever": 2412, + "french": 2413, + "london": 2414, + "center": 2415, + "six": 2416, + "red": 2417, + "2017": 2418, + "led": 2419, + "days": 2420, + "include": 2421, + "light": 2422, + "25": 2423, + "find": 2424, + "tell": 2425, + "among": 2426, + "species": 2427, + "really": 2428, + "according": 2429, + "central": 2430, + "half": 2431, + "2004": 2432, + "form": 2433, + "original": 2434, + "gave": 2435, + "office": 2436, + "making": 2437, + "enough": 2438, + "lost": 2439, + "full": 2440, + "opened": 2441, + "must": 2442, + "included": 2443, + "live": 2444, + "given": 2445, + "german": 2446, + "player": 2447, + "run": 2448, + "business": 2449, + "woman": 2450, + "community": 2451, + "cup": 2452, + "might": 2453, + "million": 2454, + "land": 2455, + "2000": 2456, + "court": 2457, + "development": 2458, + "17": 2459, + "short": 2460, + "round": 2461, + "ii": 2462, + "km": 2463, + "seen": 2464, + "class": 2465, + "story": 2466, + "always": 2467, + "become": 2468, + "sure": 2469, + "research": 2470, + "almost": 2471, + "director": 2472, + "council": 2473, + "la": 2474, + "##2": 2475, + "career": 2476, + "things": 2477, + "using": 2478, + "island": 2479, + "##z": 2480, + "couldn": 2481, + "car": 2482, + "##is": 2483, + "24": 2484, + "close": 2485, + "force": 2486, + "##1": 2487, + "better": 2488, + "free": 2489, + "support": 2490, + "control": 2491, + "field": 2492, + "students": 2493, + "2003": 2494, + "education": 2495, + "married": 2496, + "##b": 2497, + "nothing": 2498, + "worked": 2499, + "others": 2500, + "record": 2501, + "big": 2502, + "inside": 2503, + "level": 2504, + "anything": 2505, + "continued": 2506, + "give": 2507, + "james": 2508, + "##3": 2509, + "military": 2510, + "established": 2511, + "non": 2512, + "returned": 2513, + "feel": 2514, + "does": 2515, + "title": 2516, + "written": 2517, + "thing": 2518, + "feet": 2519, + "william": 2520, + "far": 2521, + "co": 2522, + "association": 2523, + "hard": 2524, + "already": 2525, + "2002": 2526, + "##ra": 2527, + "championship": 2528, + "human": 2529, + "western": 2530, + "100": 2531, + "##na": 2532, + "department": 2533, + "hall": 2534, + "role": 2535, + "various": 2536, + "production": 2537, + "21": 2538, + "19": 2539, + "heart": 2540, + "2001": 2541, + "living": 2542, + "fire": 2543, + "version": 2544, + "##ers": 2545, + "##f": 2546, + "television": 2547, + "royal": 2548, + "##4": 2549, + "produced": 2550, + "working": 2551, + "act": 2552, + "case": 2553, + "society": 2554, + "region": 2555, + "present": 2556, + "radio": 2557, + "period": 2558, + "looking": 2559, + "least": 2560, + "total": 2561, + "keep": 2562, + "england": 2563, + "wife": 2564, + "program": 2565, + "per": 2566, + "brother": 2567, + "mind": 2568, + "special": 2569, + "22": 2570, + "##le": 2571, + "am": 2572, + "works": 2573, + "soon": 2574, + "##6": 2575, + "political": 2576, + "george": 2577, + "services": 2578, + "taken": 2579, + "created": 2580, + "##7": 2581, + "further": 2582, + "able": 2583, + "reached": 2584, + "david": 2585, + "union": 2586, + "joined": 2587, + "upon": 2588, + "done": 2589, + "important": 2590, + "social": 2591, + "information": 2592, + "either": 2593, + "##ic": 2594, + "##x": 2595, + "appeared": 2596, + "position": 2597, + "ground": 2598, + "lead": 2599, + "rock": 2600, + "dark": 2601, + "election": 2602, + "23": 2603, + "board": 2604, + "france": 2605, + "hair": 2606, + "course": 2607, + "arms": 2608, + "site": 2609, + "police": 2610, + "girl": 2611, + "instead": 2612, + "real": 2613, + "sound": 2614, + "##v": 2615, + "words": 2616, + "moment": 2617, + "##te": 2618, + "someone": 2619, + "##8": 2620, + "summer": 2621, + "project": 2622, + "announced": 2623, + "san": 2624, + "less": 2625, + "wrote": 2626, + "past": 2627, + "followed": 2628, + "##5": 2629, + "blue": 2630, + "founded": 2631, + "al": 2632, + "finally": 2633, + "india": 2634, + "taking": 2635, + "records": 2636, + "america": 2637, + "##ne": 2638, + "1999": 2639, + "design": 2640, + "considered": 2641, + "northern": 2642, + "god": 2643, + "stop": 2644, + "battle": 2645, + "toward": 2646, + "european": 2647, + "outside": 2648, + "described": 2649, + "track": 2650, + "today": 2651, + "playing": 2652, + "language": 2653, + "28": 2654, + "call": 2655, + "26": 2656, + "heard": 2657, + "professional": 2658, + "low": 2659, + "australia": 2660, + "miles": 2661, + "california": 2662, + "win": 2663, + "yet": 2664, + "green": 2665, + "##ie": 2666, + "trying": 2667, + "blood": 2668, + "##ton": 2669, + "southern": 2670, + "science": 2671, + "maybe": 2672, + "everything": 2673, + "match": 2674, + "square": 2675, + "27": 2676, + "mouth": 2677, + "video": 2678, + "race": 2679, + "recorded": 2680, + "leave": 2681, + "above": 2682, + "##9": 2683, + "daughter": 2684, + "points": 2685, + "space": 2686, + "1998": 2687, + "museum": 2688, + "change": 2689, + "middle": 2690, + "common": 2691, + "##0": 2692, + "move": 2693, + "tv": 2694, + "post": 2695, + "##ta": 2696, + "lake": 2697, + "seven": 2698, + "tried": 2699, + "elected": 2700, + "closed": 2701, + "ten": 2702, + "paul": 2703, + "minister": 2704, + "##th": 2705, + "months": 2706, + "start": 2707, + "chief": 2708, + "return": 2709, + "canada": 2710, + "person": 2711, + "sea": 2712, + "release": 2713, + "similar": 2714, + "modern": 2715, + "brought": 2716, + "rest": 2717, + "hit": 2718, + "formed": 2719, + "mr": 2720, + "##la": 2721, + "1997": 2722, + "floor": 2723, + "event": 2724, + "doing": 2725, + "thomas": 2726, + "1996": 2727, + "robert": 2728, + "care": 2729, + "killed": 2730, + "training": 2731, + "star": 2732, + "week": 2733, + "needed": 2734, + "turn": 2735, + "finished": 2736, + "railway": 2737, + "rather": 2738, + "news": 2739, + "health": 2740, + "sent": 2741, + "example": 2742, + "ran": 2743, + "term": 2744, + "michael": 2745, + "coming": 2746, + "currently": 2747, + "yes": 2748, + "forces": 2749, + "despite": 2750, + "gold": 2751, + "areas": 2752, + "50": 2753, + "stage": 2754, + "fact": 2755, + "29": 2756, + "dead": 2757, + "says": 2758, + "popular": 2759, + "2018": 2760, + "originally": 2761, + "germany": 2762, + "probably": 2763, + "developed": 2764, + "result": 2765, + "pulled": 2766, + "friend": 2767, + "stood": 2768, + "money": 2769, + "running": 2770, + "mi": 2771, + "signed": 2772, + "word": 2773, + "songs": 2774, + "child": 2775, + "eventually": 2776, + "met": 2777, + "tour": 2778, + "average": 2779, + "teams": 2780, + "minutes": 2781, + "festival": 2782, + "current": 2783, + "deep": 2784, + "kind": 2785, + "1995": 2786, + "decided": 2787, + "usually": 2788, + "eastern": 2789, + "seemed": 2790, + "##ness": 2791, + "episode": 2792, + "bed": 2793, + "added": 2794, + "table": 2795, + "indian": 2796, + "private": 2797, + "charles": 2798, + "route": 2799, + "available": 2800, + "idea": 2801, + "throughout": 2802, + "centre": 2803, + "addition": 2804, + "appointed": 2805, + "style": 2806, + "1994": 2807, + "books": 2808, + "eight": 2809, + "construction": 2810, + "press": 2811, + "mean": 2812, + "wall": 2813, + "friends": 2814, + "remained": 2815, + "schools": 2816, + "study": 2817, + "##ch": 2818, + "##um": 2819, + "institute": 2820, + "oh": 2821, + "chinese": 2822, + "sometimes": 2823, + "events": 2824, + "possible": 2825, + "1992": 2826, + "australian": 2827, + "type": 2828, + "brown": 2829, + "forward": 2830, + "talk": 2831, + "process": 2832, + "food": 2833, + "debut": 2834, + "seat": 2835, + "performance": 2836, + "committee": 2837, + "features": 2838, + "character": 2839, + "arts": 2840, + "herself": 2841, + "else": 2842, + "lot": 2843, + "strong": 2844, + "russian": 2845, + "range": 2846, + "hours": 2847, + "peter": 2848, + "arm": 2849, + "##da": 2850, + "morning": 2851, + "dr": 2852, + "sold": 2853, + "##ry": 2854, + "quickly": 2855, + "directed": 2856, + "1993": 2857, + "guitar": 2858, + "china": 2859, + "##w": 2860, + "31": 2861, + "list": 2862, + "##ma": 2863, + "performed": 2864, + "media": 2865, + "uk": 2866, + "players": 2867, + "smile": 2868, + "##rs": 2869, + "myself": 2870, + "40": 2871, + "placed": 2872, + "coach": 2873, + "province": 2874, + "towards": 2875, + "wouldn": 2876, + "leading": 2877, + "whole": 2878, + "boy": 2879, + "official": 2880, + "designed": 2881, + "grand": 2882, + "census": 2883, + "##el": 2884, + "europe": 2885, + "attack": 2886, + "japanese": 2887, + "henry": 2888, + "1991": 2889, + "##re": 2890, + "##os": 2891, + "cross": 2892, + "getting": 2893, + "alone": 2894, + "action": 2895, + "lower": 2896, + "network": 2897, + "wide": 2898, + "washington": 2899, + "japan": 2900, + "1990": 2901, + "hospital": 2902, + "believe": 2903, + "changed": 2904, + "sister": 2905, + "##ar": 2906, + "hold": 2907, + "gone": 2908, + "sir": 2909, + "hadn": 2910, + "ship": 2911, + "##ka": 2912, + "studies": 2913, + "academy": 2914, + "shot": 2915, + "rights": 2916, + "below": 2917, + "base": 2918, + "bad": 2919, + "involved": 2920, + "kept": 2921, + "largest": 2922, + "##ist": 2923, + "bank": 2924, + "future": 2925, + "especially": 2926, + "beginning": 2927, + "mark": 2928, + "movement": 2929, + "section": 2930, + "female": 2931, + "magazine": 2932, + "plan": 2933, + "professor": 2934, + "lord": 2935, + "longer": 2936, + "##ian": 2937, + "sat": 2938, + "walked": 2939, + "hill": 2940, + "actually": 2941, + "civil": 2942, + "energy": 2943, + "model": 2944, + "families": 2945, + "size": 2946, + "thus": 2947, + "aircraft": 2948, + "completed": 2949, + "includes": 2950, + "data": 2951, + "captain": 2952, + "##or": 2953, + "fight": 2954, + "vocals": 2955, + "featured": 2956, + "richard": 2957, + "bridge": 2958, + "fourth": 2959, + "1989": 2960, + "officer": 2961, + "stone": 2962, + "hear": 2963, + "##ism": 2964, + "means": 2965, + "medical": 2966, + "groups": 2967, + "management": 2968, + "self": 2969, + "lips": 2970, + "competition": 2971, + "entire": 2972, + "lived": 2973, + "technology": 2974, + "leaving": 2975, + "federal": 2976, + "tournament": 2977, + "bit": 2978, + "passed": 2979, + "hot": 2980, + "independent": 2981, + "awards": 2982, + "kingdom": 2983, + "mary": 2984, + "spent": 2985, + "fine": 2986, + "doesn": 2987, + "reported": 2988, + "##ling": 2989, + "jack": 2990, + "fall": 2991, + "raised": 2992, + "itself": 2993, + "stay": 2994, + "true": 2995, + "studio": 2996, + "1988": 2997, + "sports": 2998, + "replaced": 2999, + "paris": 3000, + "systems": 3001, + "saint": 3002, + "leader": 3003, + "theatre": 3004, + "whose": 3005, + "market": 3006, + "capital": 3007, + "parents": 3008, + "spanish": 3009, + "canadian": 3010, + "earth": 3011, + "##ity": 3012, + "cut": 3013, + "degree": 3014, + "writing": 3015, + "bay": 3016, + "christian": 3017, + "awarded": 3018, + "natural": 3019, + "higher": 3020, + "bill": 3021, + "##as": 3022, + "coast": 3023, + "provided": 3024, + "previous": 3025, + "senior": 3026, + "ft": 3027, + "valley": 3028, + "organization": 3029, + "stopped": 3030, + "onto": 3031, + "countries": 3032, + "parts": 3033, + "conference": 3034, + "queen": 3035, + "security": 3036, + "interest": 3037, + "saying": 3038, + "allowed": 3039, + "master": 3040, + "earlier": 3041, + "phone": 3042, + "matter": 3043, + "smith": 3044, + "winning": 3045, + "try": 3046, + "happened": 3047, + "moving": 3048, + "campaign": 3049, + "los": 3050, + "##ley": 3051, + "breath": 3052, + "nearly": 3053, + "mid": 3054, + "1987": 3055, + "certain": 3056, + "girls": 3057, + "date": 3058, + "italian": 3059, + "african": 3060, + "standing": 3061, + "fell": 3062, + "artist": 3063, + "##ted": 3064, + "shows": 3065, + "deal": 3066, + "mine": 3067, + "industry": 3068, + "1986": 3069, + "##ng": 3070, + "everyone": 3071, + "republic": 3072, + "provide": 3073, + "collection": 3074, + "library": 3075, + "student": 3076, + "##ville": 3077, + "primary": 3078, + "owned": 3079, + "older": 3080, + "via": 3081, + "heavy": 3082, + "1st": 3083, + "makes": 3084, + "##able": 3085, + "attention": 3086, + "anyone": 3087, + "africa": 3088, + "##ri": 3089, + "stated": 3090, + "length": 3091, + "ended": 3092, + "fingers": 3093, + "command": 3094, + "staff": 3095, + "skin": 3096, + "foreign": 3097, + "opening": 3098, + "governor": 3099, + "okay": 3100, + "medal": 3101, + "kill": 3102, + "sun": 3103, + "cover": 3104, + "job": 3105, + "1985": 3106, + "introduced": 3107, + "chest": 3108, + "hell": 3109, + "feeling": 3110, + "##ies": 3111, + "success": 3112, + "meet": 3113, + "reason": 3114, + "standard": 3115, + "meeting": 3116, + "novel": 3117, + "1984": 3118, + "trade": 3119, + "source": 3120, + "buildings": 3121, + "##land": 3122, + "rose": 3123, + "guy": 3124, + "goal": 3125, + "##ur": 3126, + "chapter": 3127, + "native": 3128, + "husband": 3129, + "previously": 3130, + "unit": 3131, + "limited": 3132, + "entered": 3133, + "weeks": 3134, + "producer": 3135, + "operations": 3136, + "mountain": 3137, + "takes": 3138, + "covered": 3139, + "forced": 3140, + "related": 3141, + "roman": 3142, + "complete": 3143, + "successful": 3144, + "key": 3145, + "texas": 3146, + "cold": 3147, + "##ya": 3148, + "channel": 3149, + "1980": 3150, + "traditional": 3151, + "films": 3152, + "dance": 3153, + "clear": 3154, + "approximately": 3155, + "500": 3156, + "nine": 3157, + "van": 3158, + "prince": 3159, + "question": 3160, + "active": 3161, + "tracks": 3162, + "ireland": 3163, + "regional": 3164, + "silver": 3165, + "author": 3166, + "personal": 3167, + "sense": 3168, + "operation": 3169, + "##ine": 3170, + "economic": 3171, + "1983": 3172, + "holding": 3173, + "twenty": 3174, + "isbn": 3175, + "additional": 3176, + "speed": 3177, + "hour": 3178, + "edition": 3179, + "regular": 3180, + "historic": 3181, + "places": 3182, + "whom": 3183, + "shook": 3184, + "movie": 3185, + "km²": 3186, + "secretary": 3187, + "prior": 3188, + "report": 3189, + "chicago": 3190, + "read": 3191, + "foundation": 3192, + "view": 3193, + "engine": 3194, + "scored": 3195, + "1982": 3196, + "units": 3197, + "ask": 3198, + "airport": 3199, + "property": 3200, + "ready": 3201, + "immediately": 3202, + "lady": 3203, + "month": 3204, + "listed": 3205, + "contract": 3206, + "##de": 3207, + "manager": 3208, + "themselves": 3209, + "lines": 3210, + "##ki": 3211, + "navy": 3212, + "writer": 3213, + "meant": 3214, + "##ts": 3215, + "runs": 3216, + "##ro": 3217, + "practice": 3218, + "championships": 3219, + "singer": 3220, + "glass": 3221, + "commission": 3222, + "required": 3223, + "forest": 3224, + "starting": 3225, + "culture": 3226, + "generally": 3227, + "giving": 3228, + "access": 3229, + "attended": 3230, + "test": 3231, + "couple": 3232, + "stand": 3233, + "catholic": 3234, + "martin": 3235, + "caught": 3236, + "executive": 3237, + "##less": 3238, + "eye": 3239, + "##ey": 3240, + "thinking": 3241, + "chair": 3242, + "quite": 3243, + "shoulder": 3244, + "1979": 3245, + "hope": 3246, + "decision": 3247, + "plays": 3248, + "defeated": 3249, + "municipality": 3250, + "whether": 3251, + "structure": 3252, + "offered": 3253, + "slowly": 3254, + "pain": 3255, + "ice": 3256, + "direction": 3257, + "##ion": 3258, + "paper": 3259, + "mission": 3260, + "1981": 3261, + "mostly": 3262, + "200": 3263, + "noted": 3264, + "individual": 3265, + "managed": 3266, + "nature": 3267, + "lives": 3268, + "plant": 3269, + "##ha": 3270, + "helped": 3271, + "except": 3272, + "studied": 3273, + "computer": 3274, + "figure": 3275, + "relationship": 3276, + "issue": 3277, + "significant": 3278, + "loss": 3279, + "die": 3280, + "smiled": 3281, + "gun": 3282, + "ago": 3283, + "highest": 3284, + "1972": 3285, + "##am": 3286, + "male": 3287, + "bring": 3288, + "goals": 3289, + "mexico": 3290, + "problem": 3291, + "distance": 3292, + "commercial": 3293, + "completely": 3294, + "location": 3295, + "annual": 3296, + "famous": 3297, + "drive": 3298, + "1976": 3299, + "neck": 3300, + "1978": 3301, + "surface": 3302, + "caused": 3303, + "italy": 3304, + "understand": 3305, + "greek": 3306, + "highway": 3307, + "wrong": 3308, + "hotel": 3309, + "comes": 3310, + "appearance": 3311, + "joseph": 3312, + "double": 3313, + "issues": 3314, + "musical": 3315, + "companies": 3316, + "castle": 3317, + "income": 3318, + "review": 3319, + "assembly": 3320, + "bass": 3321, + "initially": 3322, + "parliament": 3323, + "artists": 3324, + "experience": 3325, + "1974": 3326, + "particular": 3327, + "walk": 3328, + "foot": 3329, + "engineering": 3330, + "talking": 3331, + "window": 3332, + "dropped": 3333, + "##ter": 3334, + "miss": 3335, + "baby": 3336, + "boys": 3337, + "break": 3338, + "1975": 3339, + "stars": 3340, + "edge": 3341, + "remember": 3342, + "policy": 3343, + "carried": 3344, + "train": 3345, + "stadium": 3346, + "bar": 3347, + "sex": 3348, + "angeles": 3349, + "evidence": 3350, + "##ge": 3351, + "becoming": 3352, + "assistant": 3353, + "soviet": 3354, + "1977": 3355, + "upper": 3356, + "step": 3357, + "wing": 3358, + "1970": 3359, + "youth": 3360, + "financial": 3361, + "reach": 3362, + "##ll": 3363, + "actor": 3364, + "numerous": 3365, + "##se": 3366, + "##st": 3367, + "nodded": 3368, + "arrived": 3369, + "##ation": 3370, + "minute": 3371, + "##nt": 3372, + "believed": 3373, + "sorry": 3374, + "complex": 3375, + "beautiful": 3376, + "victory": 3377, + "associated": 3378, + "temple": 3379, + "1968": 3380, + "1973": 3381, + "chance": 3382, + "perhaps": 3383, + "metal": 3384, + "##son": 3385, + "1945": 3386, + "bishop": 3387, + "##et": 3388, + "lee": 3389, + "launched": 3390, + "particularly": 3391, + "tree": 3392, + "le": 3393, + "retired": 3394, + "subject": 3395, + "prize": 3396, + "contains": 3397, + "yeah": 3398, + "theory": 3399, + "empire": 3400, + "##ce": 3401, + "suddenly": 3402, + "waiting": 3403, + "trust": 3404, + "recording": 3405, + "##to": 3406, + "happy": 3407, + "terms": 3408, + "camp": 3409, + "champion": 3410, + "1971": 3411, + "religious": 3412, + "pass": 3413, + "zealand": 3414, + "names": 3415, + "2nd": 3416, + "port": 3417, + "ancient": 3418, + "tom": 3419, + "corner": 3420, + "represented": 3421, + "watch": 3422, + "legal": 3423, + "anti": 3424, + "justice": 3425, + "cause": 3426, + "watched": 3427, + "brothers": 3428, + "45": 3429, + "material": 3430, + "changes": 3431, + "simply": 3432, + "response": 3433, + "louis": 3434, + "fast": 3435, + "##ting": 3436, + "answer": 3437, + "60": 3438, + "historical": 3439, + "1969": 3440, + "stories": 3441, + "straight": 3442, + "create": 3443, + "feature": 3444, + "increased": 3445, + "rate": 3446, + "administration": 3447, + "virginia": 3448, + "el": 3449, + "activities": 3450, + "cultural": 3451, + "overall": 3452, + "winner": 3453, + "programs": 3454, + "basketball": 3455, + "legs": 3456, + "guard": 3457, + "beyond": 3458, + "cast": 3459, + "doctor": 3460, + "mm": 3461, + "flight": 3462, + "results": 3463, + "remains": 3464, + "cost": 3465, + "effect": 3466, + "winter": 3467, + "##ble": 3468, + "larger": 3469, + "islands": 3470, + "problems": 3471, + "chairman": 3472, + "grew": 3473, + "commander": 3474, + "isn": 3475, + "1967": 3476, + "pay": 3477, + "failed": 3478, + "selected": 3479, + "hurt": 3480, + "fort": 3481, + "box": 3482, + "regiment": 3483, + "majority": 3484, + "journal": 3485, + "35": 3486, + "edward": 3487, + "plans": 3488, + "##ke": 3489, + "##ni": 3490, + "shown": 3491, + "pretty": 3492, + "irish": 3493, + "characters": 3494, + "directly": 3495, + "scene": 3496, + "likely": 3497, + "operated": 3498, + "allow": 3499, + "spring": 3500, + "##j": 3501, + "junior": 3502, + "matches": 3503, + "looks": 3504, + "mike": 3505, + "houses": 3506, + "fellow": 3507, + "##tion": 3508, + "beach": 3509, + "marriage": 3510, + "##ham": 3511, + "##ive": 3512, + "rules": 3513, + "oil": 3514, + "65": 3515, + "florida": 3516, + "expected": 3517, + "nearby": 3518, + "congress": 3519, + "sam": 3520, + "peace": 3521, + "recent": 3522, + "iii": 3523, + "wait": 3524, + "subsequently": 3525, + "cell": 3526, + "##do": 3527, + "variety": 3528, + "serving": 3529, + "agreed": 3530, + "please": 3531, + "poor": 3532, + "joe": 3533, + "pacific": 3534, + "attempt": 3535, + "wood": 3536, + "democratic": 3537, + "piece": 3538, + "prime": 3539, + "##ca": 3540, + "rural": 3541, + "mile": 3542, + "touch": 3543, + "appears": 3544, + "township": 3545, + "1964": 3546, + "1966": 3547, + "soldiers": 3548, + "##men": 3549, + "##ized": 3550, + "1965": 3551, + "pennsylvania": 3552, + "closer": 3553, + "fighting": 3554, + "claimed": 3555, + "score": 3556, + "jones": 3557, + "physical": 3558, + "editor": 3559, + "##ous": 3560, + "filled": 3561, + "genus": 3562, + "specific": 3563, + "sitting": 3564, + "super": 3565, + "mom": 3566, + "##va": 3567, + "therefore": 3568, + "supported": 3569, + "status": 3570, + "fear": 3571, + "cases": 3572, + "store": 3573, + "meaning": 3574, + "wales": 3575, + "minor": 3576, + "spain": 3577, + "tower": 3578, + "focus": 3579, + "vice": 3580, + "frank": 3581, + "follow": 3582, + "parish": 3583, + "separate": 3584, + "golden": 3585, + "horse": 3586, + "fifth": 3587, + "remaining": 3588, + "branch": 3589, + "32": 3590, + "presented": 3591, + "stared": 3592, + "##id": 3593, + "uses": 3594, + "secret": 3595, + "forms": 3596, + "##co": 3597, + "baseball": 3598, + "exactly": 3599, + "##ck": 3600, + "choice": 3601, + "note": 3602, + "discovered": 3603, + "travel": 3604, + "composed": 3605, + "truth": 3606, + "russia": 3607, + "ball": 3608, + "color": 3609, + "kiss": 3610, + "dad": 3611, + "wind": 3612, + "continue": 3613, + "ring": 3614, + "referred": 3615, + "numbers": 3616, + "digital": 3617, + "greater": 3618, + "##ns": 3619, + "metres": 3620, + "slightly": 3621, + "direct": 3622, + "increase": 3623, + "1960": 3624, + "responsible": 3625, + "crew": 3626, + "rule": 3627, + "trees": 3628, + "troops": 3629, + "##no": 3630, + "broke": 3631, + "goes": 3632, + "individuals": 3633, + "hundred": 3634, + "weight": 3635, + "creek": 3636, + "sleep": 3637, + "memory": 3638, + "defense": 3639, + "provides": 3640, + "ordered": 3641, + "code": 3642, + "value": 3643, + "jewish": 3644, + "windows": 3645, + "1944": 3646, + "safe": 3647, + "judge": 3648, + "whatever": 3649, + "corps": 3650, + "realized": 3651, + "growing": 3652, + "pre": 3653, + "##ga": 3654, + "cities": 3655, + "alexander": 3656, + "gaze": 3657, + "lies": 3658, + "spread": 3659, + "scott": 3660, + "letter": 3661, + "showed": 3662, + "situation": 3663, + "mayor": 3664, + "transport": 3665, + "watching": 3666, + "workers": 3667, + "extended": 3668, + "##li": 3669, + "expression": 3670, + "normal": 3671, + "##ment": 3672, + "chart": 3673, + "multiple": 3674, + "border": 3675, + "##ba": 3676, + "host": 3677, + "##ner": 3678, + "daily": 3679, + "mrs": 3680, + "walls": 3681, + "piano": 3682, + "##ko": 3683, + "heat": 3684, + "cannot": 3685, + "##ate": 3686, + "earned": 3687, + "products": 3688, + "drama": 3689, + "era": 3690, + "authority": 3691, + "seasons": 3692, + "join": 3693, + "grade": 3694, + "##io": 3695, + "sign": 3696, + "difficult": 3697, + "machine": 3698, + "1963": 3699, + "territory": 3700, + "mainly": 3701, + "##wood": 3702, + "stations": 3703, + "squadron": 3704, + "1962": 3705, + "stepped": 3706, + "iron": 3707, + "19th": 3708, + "##led": 3709, + "serve": 3710, + "appear": 3711, + "sky": 3712, + "speak": 3713, + "broken": 3714, + "charge": 3715, + "knowledge": 3716, + "kilometres": 3717, + "removed": 3718, + "ships": 3719, + "article": 3720, + "campus": 3721, + "simple": 3722, + "##ty": 3723, + "pushed": 3724, + "britain": 3725, + "##ve": 3726, + "leaves": 3727, + "recently": 3728, + "cd": 3729, + "soft": 3730, + "boston": 3731, + "latter": 3732, + "easy": 3733, + "acquired": 3734, + "poland": 3735, + "##sa": 3736, + "quality": 3737, + "officers": 3738, + "presence": 3739, + "planned": 3740, + "nations": 3741, + "mass": 3742, + "broadcast": 3743, + "jean": 3744, + "share": 3745, + "image": 3746, + "influence": 3747, + "wild": 3748, + "offer": 3749, + "emperor": 3750, + "electric": 3751, + "reading": 3752, + "headed": 3753, + "ability": 3754, + "promoted": 3755, + "yellow": 3756, + "ministry": 3757, + "1942": 3758, + "throat": 3759, + "smaller": 3760, + "politician": 3761, + "##by": 3762, + "latin": 3763, + "spoke": 3764, + "cars": 3765, + "williams": 3766, + "males": 3767, + "lack": 3768, + "pop": 3769, + "80": 3770, + "##ier": 3771, + "acting": 3772, + "seeing": 3773, + "consists": 3774, + "##ti": 3775, + "estate": 3776, + "1961": 3777, + "pressure": 3778, + "johnson": 3779, + "newspaper": 3780, + "jr": 3781, + "chris": 3782, + "olympics": 3783, + "online": 3784, + "conditions": 3785, + "beat": 3786, + "elements": 3787, + "walking": 3788, + "vote": 3789, + "##field": 3790, + "needs": 3791, + "carolina": 3792, + "text": 3793, + "featuring": 3794, + "global": 3795, + "block": 3796, + "shirt": 3797, + "levels": 3798, + "francisco": 3799, + "purpose": 3800, + "females": 3801, + "et": 3802, + "dutch": 3803, + "duke": 3804, + "ahead": 3805, + "gas": 3806, + "twice": 3807, + "safety": 3808, + "serious": 3809, + "turning": 3810, + "highly": 3811, + "lieutenant": 3812, + "firm": 3813, + "maria": 3814, + "amount": 3815, + "mixed": 3816, + "daniel": 3817, + "proposed": 3818, + "perfect": 3819, + "agreement": 3820, + "affairs": 3821, + "3rd": 3822, + "seconds": 3823, + "contemporary": 3824, + "paid": 3825, + "1943": 3826, + "prison": 3827, + "save": 3828, + "kitchen": 3829, + "label": 3830, + "administrative": 3831, + "intended": 3832, + "constructed": 3833, + "academic": 3834, + "nice": 3835, + "teacher": 3836, + "races": 3837, + "1956": 3838, + "formerly": 3839, + "corporation": 3840, + "ben": 3841, + "nation": 3842, + "issued": 3843, + "shut": 3844, + "1958": 3845, + "drums": 3846, + "housing": 3847, + "victoria": 3848, + "seems": 3849, + "opera": 3850, + "1959": 3851, + "graduated": 3852, + "function": 3853, + "von": 3854, + "mentioned": 3855, + "picked": 3856, + "build": 3857, + "recognized": 3858, + "shortly": 3859, + "protection": 3860, + "picture": 3861, + "notable": 3862, + "exchange": 3863, + "elections": 3864, + "1980s": 3865, + "loved": 3866, + "percent": 3867, + "racing": 3868, + "fish": 3869, + "elizabeth": 3870, + "garden": 3871, + "volume": 3872, + "hockey": 3873, + "1941": 3874, + "beside": 3875, + "settled": 3876, + "##ford": 3877, + "1940": 3878, + "competed": 3879, + "replied": 3880, + "drew": 3881, + "1948": 3882, + "actress": 3883, + "marine": 3884, + "scotland": 3885, + "steel": 3886, + "glanced": 3887, + "farm": 3888, + "steve": 3889, + "1957": 3890, + "risk": 3891, + "tonight": 3892, + "positive": 3893, + "magic": 3894, + "singles": 3895, + "effects": 3896, + "gray": 3897, + "screen": 3898, + "dog": 3899, + "##ja": 3900, + "residents": 3901, + "bus": 3902, + "sides": 3903, + "none": 3904, + "secondary": 3905, + "literature": 3906, + "polish": 3907, + "destroyed": 3908, + "flying": 3909, + "founder": 3910, + "households": 3911, + "1939": 3912, + "lay": 3913, + "reserve": 3914, + "usa": 3915, + "gallery": 3916, + "##ler": 3917, + "1946": 3918, + "industrial": 3919, + "younger": 3920, + "approach": 3921, + "appearances": 3922, + "urban": 3923, + "ones": 3924, + "1950": 3925, + "finish": 3926, + "avenue": 3927, + "powerful": 3928, + "fully": 3929, + "growth": 3930, + "page": 3931, + "honor": 3932, + "jersey": 3933, + "projects": 3934, + "advanced": 3935, + "revealed": 3936, + "basic": 3937, + "90": 3938, + "infantry": 3939, + "pair": 3940, + "equipment": 3941, + "visit": 3942, + "33": 3943, + "evening": 3944, + "search": 3945, + "grant": 3946, + "effort": 3947, + "solo": 3948, + "treatment": 3949, + "buried": 3950, + "republican": 3951, + "primarily": 3952, + "bottom": 3953, + "owner": 3954, + "1970s": 3955, + "israel": 3956, + "gives": 3957, + "jim": 3958, + "dream": 3959, + "bob": 3960, + "remain": 3961, + "spot": 3962, + "70": 3963, + "notes": 3964, + "produce": 3965, + "champions": 3966, + "contact": 3967, + "ed": 3968, + "soul": 3969, + "accepted": 3970, + "ways": 3971, + "del": 3972, + "##ally": 3973, + "losing": 3974, + "split": 3975, + "price": 3976, + "capacity": 3977, + "basis": 3978, + "trial": 3979, + "questions": 3980, + "##ina": 3981, + "1955": 3982, + "20th": 3983, + "guess": 3984, + "officially": 3985, + "memorial": 3986, + "naval": 3987, + "initial": 3988, + "##ization": 3989, + "whispered": 3990, + "median": 3991, + "engineer": 3992, + "##ful": 3993, + "sydney": 3994, + "##go": 3995, + "columbia": 3996, + "strength": 3997, + "300": 3998, + "1952": 3999, + "tears": 4000, + "senate": 4001, + "00": 4002, + "card": 4003, + "asian": 4004, + "agent": 4005, + "1947": 4006, + "software": 4007, + "44": 4008, + "draw": 4009, + "warm": 4010, + "supposed": 4011, + "com": 4012, + "pro": 4013, + "##il": 4014, + "transferred": 4015, + "leaned": 4016, + "##at": 4017, + "candidate": 4018, + "escape": 4019, + "mountains": 4020, + "asia": 4021, + "potential": 4022, + "activity": 4023, + "entertainment": 4024, + "seem": 4025, + "traffic": 4026, + "jackson": 4027, + "murder": 4028, + "36": 4029, + "slow": 4030, + "product": 4031, + "orchestra": 4032, + "haven": 4033, + "agency": 4034, + "bbc": 4035, + "taught": 4036, + "website": 4037, + "comedy": 4038, + "unable": 4039, + "storm": 4040, + "planning": 4041, + "albums": 4042, + "rugby": 4043, + "environment": 4044, + "scientific": 4045, + "grabbed": 4046, + "protect": 4047, + "##hi": 4048, + "boat": 4049, + "typically": 4050, + "1954": 4051, + "1953": 4052, + "damage": 4053, + "principal": 4054, + "divided": 4055, + "dedicated": 4056, + "mount": 4057, + "ohio": 4058, + "##berg": 4059, + "pick": 4060, + "fought": 4061, + "driver": 4062, + "##der": 4063, + "empty": 4064, + "shoulders": 4065, + "sort": 4066, + "thank": 4067, + "berlin": 4068, + "prominent": 4069, + "account": 4070, + "freedom": 4071, + "necessary": 4072, + "efforts": 4073, + "alex": 4074, + "headquarters": 4075, + "follows": 4076, + "alongside": 4077, + "des": 4078, + "simon": 4079, + "andrew": 4080, + "suggested": 4081, + "operating": 4082, + "learning": 4083, + "steps": 4084, + "1949": 4085, + "sweet": 4086, + "technical": 4087, + "begin": 4088, + "easily": 4089, + "34": 4090, + "teeth": 4091, + "speaking": 4092, + "settlement": 4093, + "scale": 4094, + "##sh": 4095, + "renamed": 4096, + "ray": 4097, + "max": 4098, + "enemy": 4099, + "semi": 4100, + "joint": 4101, + "compared": 4102, + "##rd": 4103, + "scottish": 4104, + "leadership": 4105, + "analysis": 4106, + "offers": 4107, + "georgia": 4108, + "pieces": 4109, + "captured": 4110, + "animal": 4111, + "deputy": 4112, + "guest": 4113, + "organized": 4114, + "##lin": 4115, + "tony": 4116, + "combined": 4117, + "method": 4118, + "challenge": 4119, + "1960s": 4120, + "huge": 4121, + "wants": 4122, + "battalion": 4123, + "sons": 4124, + "rise": 4125, + "crime": 4126, + "types": 4127, + "facilities": 4128, + "telling": 4129, + "path": 4130, + "1951": 4131, + "platform": 4132, + "sit": 4133, + "1990s": 4134, + "##lo": 4135, + "tells": 4136, + "assigned": 4137, + "rich": 4138, + "pull": 4139, + "##ot": 4140, + "commonly": 4141, + "alive": 4142, + "##za": 4143, + "letters": 4144, + "concept": 4145, + "conducted": 4146, + "wearing": 4147, + "happen": 4148, + "bought": 4149, + "becomes": 4150, + "holy": 4151, + "gets": 4152, + "ocean": 4153, + "defeat": 4154, + "languages": 4155, + "purchased": 4156, + "coffee": 4157, + "occurred": 4158, + "titled": 4159, + "##q": 4160, + "declared": 4161, + "applied": 4162, + "sciences": 4163, + "concert": 4164, + "sounds": 4165, + "jazz": 4166, + "brain": 4167, + "##me": 4168, + "painting": 4169, + "fleet": 4170, + "tax": 4171, + "nick": 4172, + "##ius": 4173, + "michigan": 4174, + "count": 4175, + "animals": 4176, + "leaders": 4177, + "episodes": 4178, + "##line": 4179, + "content": 4180, + "##den": 4181, + "birth": 4182, + "##it": 4183, + "clubs": 4184, + "64": 4185, + "palace": 4186, + "critical": 4187, + "refused": 4188, + "fair": 4189, + "leg": 4190, + "laughed": 4191, + "returning": 4192, + "surrounding": 4193, + "participated": 4194, + "formation": 4195, + "lifted": 4196, + "pointed": 4197, + "connected": 4198, + "rome": 4199, + "medicine": 4200, + "laid": 4201, + "taylor": 4202, + "santa": 4203, + "powers": 4204, + "adam": 4205, + "tall": 4206, + "shared": 4207, + "focused": 4208, + "knowing": 4209, + "yards": 4210, + "entrance": 4211, + "falls": 4212, + "##wa": 4213, + "calling": 4214, + "##ad": 4215, + "sources": 4216, + "chosen": 4217, + "beneath": 4218, + "resources": 4219, + "yard": 4220, + "##ite": 4221, + "nominated": 4222, + "silence": 4223, + "zone": 4224, + "defined": 4225, + "##que": 4226, + "gained": 4227, + "thirty": 4228, + "38": 4229, + "bodies": 4230, + "moon": 4231, + "##ard": 4232, + "adopted": 4233, + "christmas": 4234, + "widely": 4235, + "register": 4236, + "apart": 4237, + "iran": 4238, + "premier": 4239, + "serves": 4240, + "du": 4241, + "unknown": 4242, + "parties": 4243, + "##les": 4244, + "generation": 4245, + "##ff": 4246, + "continues": 4247, + "quick": 4248, + "fields": 4249, + "brigade": 4250, + "quiet": 4251, + "teaching": 4252, + "clothes": 4253, + "impact": 4254, + "weapons": 4255, + "partner": 4256, + "flat": 4257, + "theater": 4258, + "supreme": 4259, + "1938": 4260, + "37": 4261, + "relations": 4262, + "##tor": 4263, + "plants": 4264, + "suffered": 4265, + "1936": 4266, + "wilson": 4267, + "kids": 4268, + "begins": 4269, + "##age": 4270, + "1918": 4271, + "seats": 4272, + "armed": 4273, + "internet": 4274, + "models": 4275, + "worth": 4276, + "laws": 4277, + "400": 4278, + "communities": 4279, + "classes": 4280, + "background": 4281, + "knows": 4282, + "thanks": 4283, + "quarter": 4284, + "reaching": 4285, + "humans": 4286, + "carry": 4287, + "killing": 4288, + "format": 4289, + "kong": 4290, + "hong": 4291, + "setting": 4292, + "75": 4293, + "architecture": 4294, + "disease": 4295, + "railroad": 4296, + "inc": 4297, + "possibly": 4298, + "wish": 4299, + "arthur": 4300, + "thoughts": 4301, + "harry": 4302, + "doors": 4303, + "density": 4304, + "##di": 4305, + "crowd": 4306, + "illinois": 4307, + "stomach": 4308, + "tone": 4309, + "unique": 4310, + "reports": 4311, + "anyway": 4312, + "##ir": 4313, + "liberal": 4314, + "der": 4315, + "vehicle": 4316, + "thick": 4317, + "dry": 4318, + "drug": 4319, + "faced": 4320, + "largely": 4321, + "facility": 4322, + "theme": 4323, + "holds": 4324, + "creation": 4325, + "strange": 4326, + "colonel": 4327, + "##mi": 4328, + "revolution": 4329, + "bell": 4330, + "politics": 4331, + "turns": 4332, + "silent": 4333, + "rail": 4334, + "relief": 4335, + "independence": 4336, + "combat": 4337, + "shape": 4338, + "write": 4339, + "determined": 4340, + "sales": 4341, + "learned": 4342, + "4th": 4343, + "finger": 4344, + "oxford": 4345, + "providing": 4346, + "1937": 4347, + "heritage": 4348, + "fiction": 4349, + "situated": 4350, + "designated": 4351, + "allowing": 4352, + "distribution": 4353, + "hosted": 4354, + "##est": 4355, + "sight": 4356, + "interview": 4357, + "estimated": 4358, + "reduced": 4359, + "##ria": 4360, + "toronto": 4361, + "footballer": 4362, + "keeping": 4363, + "guys": 4364, + "damn": 4365, + "claim": 4366, + "motion": 4367, + "sport": 4368, + "sixth": 4369, + "stayed": 4370, + "##ze": 4371, + "en": 4372, + "rear": 4373, + "receive": 4374, + "handed": 4375, + "twelve": 4376, + "dress": 4377, + "audience": 4378, + "granted": 4379, + "brazil": 4380, + "##well": 4381, + "spirit": 4382, + "##ated": 4383, + "noticed": 4384, + "etc": 4385, + "olympic": 4386, + "representative": 4387, + "eric": 4388, + "tight": 4389, + "trouble": 4390, + "reviews": 4391, + "drink": 4392, + "vampire": 4393, + "missing": 4394, + "roles": 4395, + "ranked": 4396, + "newly": 4397, + "household": 4398, + "finals": 4399, + "wave": 4400, + "critics": 4401, + "##ee": 4402, + "phase": 4403, + "massachusetts": 4404, + "pilot": 4405, + "unlike": 4406, + "philadelphia": 4407, + "bright": 4408, + "guns": 4409, + "crown": 4410, + "organizations": 4411, + "roof": 4412, + "42": 4413, + "respectively": 4414, + "clearly": 4415, + "tongue": 4416, + "marked": 4417, + "circle": 4418, + "fox": 4419, + "korea": 4420, + "bronze": 4421, + "brian": 4422, + "expanded": 4423, + "sexual": 4424, + "supply": 4425, + "yourself": 4426, + "inspired": 4427, + "labour": 4428, + "fc": 4429, + "##ah": 4430, + "reference": 4431, + "vision": 4432, + "draft": 4433, + "connection": 4434, + "brand": 4435, + "reasons": 4436, + "1935": 4437, + "classic": 4438, + "driving": 4439, + "trip": 4440, + "jesus": 4441, + "cells": 4442, + "entry": 4443, + "1920": 4444, + "neither": 4445, + "trail": 4446, + "claims": 4447, + "atlantic": 4448, + "orders": 4449, + "labor": 4450, + "nose": 4451, + "afraid": 4452, + "identified": 4453, + "intelligence": 4454, + "calls": 4455, + "cancer": 4456, + "attacked": 4457, + "passing": 4458, + "stephen": 4459, + "positions": 4460, + "imperial": 4461, + "grey": 4462, + "jason": 4463, + "39": 4464, + "sunday": 4465, + "48": 4466, + "swedish": 4467, + "avoid": 4468, + "extra": 4469, + "uncle": 4470, + "message": 4471, + "covers": 4472, + "allows": 4473, + "surprise": 4474, + "materials": 4475, + "fame": 4476, + "hunter": 4477, + "##ji": 4478, + "1930": 4479, + "citizens": 4480, + "figures": 4481, + "davis": 4482, + "environmental": 4483, + "confirmed": 4484, + "shit": 4485, + "titles": 4486, + "di": 4487, + "performing": 4488, + "difference": 4489, + "acts": 4490, + "attacks": 4491, + "##ov": 4492, + "existing": 4493, + "votes": 4494, + "opportunity": 4495, + "nor": 4496, + "shop": 4497, + "entirely": 4498, + "trains": 4499, + "opposite": 4500, + "pakistan": 4501, + "##pa": 4502, + "develop": 4503, + "resulted": 4504, + "representatives": 4505, + "actions": 4506, + "reality": 4507, + "pressed": 4508, + "##ish": 4509, + "barely": 4510, + "wine": 4511, + "conversation": 4512, + "faculty": 4513, + "northwest": 4514, + "ends": 4515, + "documentary": 4516, + "nuclear": 4517, + "stock": 4518, + "grace": 4519, + "sets": 4520, + "eat": 4521, + "alternative": 4522, + "##ps": 4523, + "bag": 4524, + "resulting": 4525, + "creating": 4526, + "surprised": 4527, + "cemetery": 4528, + "1919": 4529, + "drop": 4530, + "finding": 4531, + "sarah": 4532, + "cricket": 4533, + "streets": 4534, + "tradition": 4535, + "ride": 4536, + "1933": 4537, + "exhibition": 4538, + "target": 4539, + "ear": 4540, + "explained": 4541, + "rain": 4542, + "composer": 4543, + "injury": 4544, + "apartment": 4545, + "municipal": 4546, + "educational": 4547, + "occupied": 4548, + "netherlands": 4549, + "clean": 4550, + "billion": 4551, + "constitution": 4552, + "learn": 4553, + "1914": 4554, + "maximum": 4555, + "classical": 4556, + "francis": 4557, + "lose": 4558, + "opposition": 4559, + "jose": 4560, + "ontario": 4561, + "bear": 4562, + "core": 4563, + "hills": 4564, + "rolled": 4565, + "ending": 4566, + "drawn": 4567, + "permanent": 4568, + "fun": 4569, + "##tes": 4570, + "##lla": 4571, + "lewis": 4572, + "sites": 4573, + "chamber": 4574, + "ryan": 4575, + "##way": 4576, + "scoring": 4577, + "height": 4578, + "1934": 4579, + "##house": 4580, + "lyrics": 4581, + "staring": 4582, + "55": 4583, + "officials": 4584, + "1917": 4585, + "snow": 4586, + "oldest": 4587, + "##tic": 4588, + "orange": 4589, + "##ger": 4590, + "qualified": 4591, + "interior": 4592, + "apparently": 4593, + "succeeded": 4594, + "thousand": 4595, + "dinner": 4596, + "lights": 4597, + "existence": 4598, + "fans": 4599, + "heavily": 4600, + "41": 4601, + "greatest": 4602, + "conservative": 4603, + "send": 4604, + "bowl": 4605, + "plus": 4606, + "enter": 4607, + "catch": 4608, + "##un": 4609, + "economy": 4610, + "duty": 4611, + "1929": 4612, + "speech": 4613, + "authorities": 4614, + "princess": 4615, + "performances": 4616, + "versions": 4617, + "shall": 4618, + "graduate": 4619, + "pictures": 4620, + "effective": 4621, + "remembered": 4622, + "poetry": 4623, + "desk": 4624, + "crossed": 4625, + "starring": 4626, + "starts": 4627, + "passenger": 4628, + "sharp": 4629, + "##ant": 4630, + "acres": 4631, + "ass": 4632, + "weather": 4633, + "falling": 4634, + "rank": 4635, + "fund": 4636, + "supporting": 4637, + "check": 4638, + "adult": 4639, + "publishing": 4640, + "heads": 4641, + "cm": 4642, + "southeast": 4643, + "lane": 4644, + "##burg": 4645, + "application": 4646, + "bc": 4647, + "##ura": 4648, + "les": 4649, + "condition": 4650, + "transfer": 4651, + "prevent": 4652, + "display": 4653, + "ex": 4654, + "regions": 4655, + "earl": 4656, + "federation": 4657, + "cool": 4658, + "relatively": 4659, + "answered": 4660, + "besides": 4661, + "1928": 4662, + "obtained": 4663, + "portion": 4664, + "##town": 4665, + "mix": 4666, + "##ding": 4667, + "reaction": 4668, + "liked": 4669, + "dean": 4670, + "express": 4671, + "peak": 4672, + "1932": 4673, + "##tte": 4674, + "counter": 4675, + "religion": 4676, + "chain": 4677, + "rare": 4678, + "miller": 4679, + "convention": 4680, + "aid": 4681, + "lie": 4682, + "vehicles": 4683, + "mobile": 4684, + "perform": 4685, + "squad": 4686, + "wonder": 4687, + "lying": 4688, + "crazy": 4689, + "sword": 4690, + "##ping": 4691, + "attempted": 4692, + "centuries": 4693, + "weren": 4694, + "philosophy": 4695, + "category": 4696, + "##ize": 4697, + "anna": 4698, + "interested": 4699, + "47": 4700, + "sweden": 4701, + "wolf": 4702, + "frequently": 4703, + "abandoned": 4704, + "kg": 4705, + "literary": 4706, + "alliance": 4707, + "task": 4708, + "entitled": 4709, + "##ay": 4710, + "threw": 4711, + "promotion": 4712, + "factory": 4713, + "tiny": 4714, + "soccer": 4715, + "visited": 4716, + "matt": 4717, + "fm": 4718, + "achieved": 4719, + "52": 4720, + "defence": 4721, + "internal": 4722, + "persian": 4723, + "43": 4724, + "methods": 4725, + "##ging": 4726, + "arrested": 4727, + "otherwise": 4728, + "cambridge": 4729, + "programming": 4730, + "villages": 4731, + "elementary": 4732, + "districts": 4733, + "rooms": 4734, + "criminal": 4735, + "conflict": 4736, + "worry": 4737, + "trained": 4738, + "1931": 4739, + "attempts": 4740, + "waited": 4741, + "signal": 4742, + "bird": 4743, + "truck": 4744, + "subsequent": 4745, + "programme": 4746, + "##ol": 4747, + "ad": 4748, + "49": 4749, + "communist": 4750, + "details": 4751, + "faith": 4752, + "sector": 4753, + "patrick": 4754, + "carrying": 4755, + "laugh": 4756, + "##ss": 4757, + "controlled": 4758, + "korean": 4759, + "showing": 4760, + "origin": 4761, + "fuel": 4762, + "evil": 4763, + "1927": 4764, + "##ent": 4765, + "brief": 4766, + "identity": 4767, + "darkness": 4768, + "address": 4769, + "pool": 4770, + "missed": 4771, + "publication": 4772, + "web": 4773, + "planet": 4774, + "ian": 4775, + "anne": 4776, + "wings": 4777, + "invited": 4778, + "##tt": 4779, + "briefly": 4780, + "standards": 4781, + "kissed": 4782, + "##be": 4783, + "ideas": 4784, + "climate": 4785, + "causing": 4786, + "walter": 4787, + "worse": 4788, + "albert": 4789, + "articles": 4790, + "winners": 4791, + "desire": 4792, + "aged": 4793, + "northeast": 4794, + "dangerous": 4795, + "gate": 4796, + "doubt": 4797, + "1922": 4798, + "wooden": 4799, + "multi": 4800, + "##ky": 4801, + "poet": 4802, + "rising": 4803, + "funding": 4804, + "46": 4805, + "communications": 4806, + "communication": 4807, + "violence": 4808, + "copies": 4809, + "prepared": 4810, + "ford": 4811, + "investigation": 4812, + "skills": 4813, + "1924": 4814, + "pulling": 4815, + "electronic": 4816, + "##ak": 4817, + "##ial": 4818, + "##han": 4819, + "containing": 4820, + "ultimately": 4821, + "offices": 4822, + "singing": 4823, + "understanding": 4824, + "restaurant": 4825, + "tomorrow": 4826, + "fashion": 4827, + "christ": 4828, + "ward": 4829, + "da": 4830, + "pope": 4831, + "stands": 4832, + "5th": 4833, + "flow": 4834, + "studios": 4835, + "aired": 4836, + "commissioned": 4837, + "contained": 4838, + "exist": 4839, + "fresh": 4840, + "americans": 4841, + "##per": 4842, + "wrestling": 4843, + "approved": 4844, + "kid": 4845, + "employed": 4846, + "respect": 4847, + "suit": 4848, + "1925": 4849, + "angel": 4850, + "asking": 4851, + "increasing": 4852, + "frame": 4853, + "angry": 4854, + "selling": 4855, + "1950s": 4856, + "thin": 4857, + "finds": 4858, + "##nd": 4859, + "temperature": 4860, + "statement": 4861, + "ali": 4862, + "explain": 4863, + "inhabitants": 4864, + "towns": 4865, + "extensive": 4866, + "narrow": 4867, + "51": 4868, + "jane": 4869, + "flowers": 4870, + "images": 4871, + "promise": 4872, + "somewhere": 4873, + "object": 4874, + "fly": 4875, + "closely": 4876, + "##ls": 4877, + "1912": 4878, + "bureau": 4879, + "cape": 4880, + "1926": 4881, + "weekly": 4882, + "presidential": 4883, + "legislative": 4884, + "1921": 4885, + "##ai": 4886, + "##au": 4887, + "launch": 4888, + "founding": 4889, + "##ny": 4890, + "978": 4891, + "##ring": 4892, + "artillery": 4893, + "strike": 4894, + "un": 4895, + "institutions": 4896, + "roll": 4897, + "writers": 4898, + "landing": 4899, + "chose": 4900, + "kevin": 4901, + "anymore": 4902, + "pp": 4903, + "##ut": 4904, + "attorney": 4905, + "fit": 4906, + "dan": 4907, + "billboard": 4908, + "receiving": 4909, + "agricultural": 4910, + "breaking": 4911, + "sought": 4912, + "dave": 4913, + "admitted": 4914, + "lands": 4915, + "mexican": 4916, + "##bury": 4917, + "charlie": 4918, + "specifically": 4919, + "hole": 4920, + "iv": 4921, + "howard": 4922, + "credit": 4923, + "moscow": 4924, + "roads": 4925, + "accident": 4926, + "1923": 4927, + "proved": 4928, + "wear": 4929, + "struck": 4930, + "hey": 4931, + "guards": 4932, + "stuff": 4933, + "slid": 4934, + "expansion": 4935, + "1915": 4936, + "cat": 4937, + "anthony": 4938, + "##kin": 4939, + "melbourne": 4940, + "opposed": 4941, + "sub": 4942, + "southwest": 4943, + "architect": 4944, + "failure": 4945, + "plane": 4946, + "1916": 4947, + "##ron": 4948, + "map": 4949, + "camera": 4950, + "tank": 4951, + "listen": 4952, + "regarding": 4953, + "wet": 4954, + "introduction": 4955, + "metropolitan": 4956, + "link": 4957, + "ep": 4958, + "fighter": 4959, + "inch": 4960, + "grown": 4961, + "gene": 4962, + "anger": 4963, + "fixed": 4964, + "buy": 4965, + "dvd": 4966, + "khan": 4967, + "domestic": 4968, + "worldwide": 4969, + "chapel": 4970, + "mill": 4971, + "functions": 4972, + "examples": 4973, + "##head": 4974, + "developing": 4975, + "1910": 4976, + "turkey": 4977, + "hits": 4978, + "pocket": 4979, + "antonio": 4980, + "papers": 4981, + "grow": 4982, + "unless": 4983, + "circuit": 4984, + "18th": 4985, + "concerned": 4986, + "attached": 4987, + "journalist": 4988, + "selection": 4989, + "journey": 4990, + "converted": 4991, + "provincial": 4992, + "painted": 4993, + "hearing": 4994, + "aren": 4995, + "bands": 4996, + "negative": 4997, + "aside": 4998, + "wondered": 4999, + "knight": 5000, + "lap": 5001, + "survey": 5002, + "ma": 5003, + "##ow": 5004, + "noise": 5005, + "billy": 5006, + "##ium": 5007, + "shooting": 5008, + "guide": 5009, + "bedroom": 5010, + "priest": 5011, + "resistance": 5012, + "motor": 5013, + "homes": 5014, + "sounded": 5015, + "giant": 5016, + "##mer": 5017, + "150": 5018, + "scenes": 5019, + "equal": 5020, + "comic": 5021, + "patients": 5022, + "hidden": 5023, + "solid": 5024, + "actual": 5025, + "bringing": 5026, + "afternoon": 5027, + "touched": 5028, + "funds": 5029, + "wedding": 5030, + "consisted": 5031, + "marie": 5032, + "canal": 5033, + "sr": 5034, + "kim": 5035, + "treaty": 5036, + "turkish": 5037, + "recognition": 5038, + "residence": 5039, + "cathedral": 5040, + "broad": 5041, + "knees": 5042, + "incident": 5043, + "shaped": 5044, + "fired": 5045, + "norwegian": 5046, + "handle": 5047, + "cheek": 5048, + "contest": 5049, + "represent": 5050, + "##pe": 5051, + "representing": 5052, + "beauty": 5053, + "##sen": 5054, + "birds": 5055, + "advantage": 5056, + "emergency": 5057, + "wrapped": 5058, + "drawing": 5059, + "notice": 5060, + "pink": 5061, + "broadcasting": 5062, + "##ong": 5063, + "somehow": 5064, + "bachelor": 5065, + "seventh": 5066, + "collected": 5067, + "registered": 5068, + "establishment": 5069, + "alan": 5070, + "assumed": 5071, + "chemical": 5072, + "personnel": 5073, + "roger": 5074, + "retirement": 5075, + "jeff": 5076, + "portuguese": 5077, + "wore": 5078, + "tied": 5079, + "device": 5080, + "threat": 5081, + "progress": 5082, + "advance": 5083, + "##ised": 5084, + "banks": 5085, + "hired": 5086, + "manchester": 5087, + "nfl": 5088, + "teachers": 5089, + "structures": 5090, + "forever": 5091, + "##bo": 5092, + "tennis": 5093, + "helping": 5094, + "saturday": 5095, + "sale": 5096, + "applications": 5097, + "junction": 5098, + "hip": 5099, + "incorporated": 5100, + "neighborhood": 5101, + "dressed": 5102, + "ceremony": 5103, + "##ds": 5104, + "influenced": 5105, + "hers": 5106, + "visual": 5107, + "stairs": 5108, + "decades": 5109, + "inner": 5110, + "kansas": 5111, + "hung": 5112, + "hoped": 5113, + "gain": 5114, + "scheduled": 5115, + "downtown": 5116, + "engaged": 5117, + "austria": 5118, + "clock": 5119, + "norway": 5120, + "certainly": 5121, + "pale": 5122, + "protected": 5123, + "1913": 5124, + "victor": 5125, + "employees": 5126, + "plate": 5127, + "putting": 5128, + "surrounded": 5129, + "##ists": 5130, + "finishing": 5131, + "blues": 5132, + "tropical": 5133, + "##ries": 5134, + "minnesota": 5135, + "consider": 5136, + "philippines": 5137, + "accept": 5138, + "54": 5139, + "retrieved": 5140, + "1900": 5141, + "concern": 5142, + "anderson": 5143, + "properties": 5144, + "institution": 5145, + "gordon": 5146, + "successfully": 5147, + "vietnam": 5148, + "##dy": 5149, + "backing": 5150, + "outstanding": 5151, + "muslim": 5152, + "crossing": 5153, + "folk": 5154, + "producing": 5155, + "usual": 5156, + "demand": 5157, + "occurs": 5158, + "observed": 5159, + "lawyer": 5160, + "educated": 5161, + "##ana": 5162, + "kelly": 5163, + "string": 5164, + "pleasure": 5165, + "budget": 5166, + "items": 5167, + "quietly": 5168, + "colorado": 5169, + "philip": 5170, + "typical": 5171, + "##worth": 5172, + "derived": 5173, + "600": 5174, + "survived": 5175, + "asks": 5176, + "mental": 5177, + "##ide": 5178, + "56": 5179, + "jake": 5180, + "jews": 5181, + "distinguished": 5182, + "ltd": 5183, + "1911": 5184, + "sri": 5185, + "extremely": 5186, + "53": 5187, + "athletic": 5188, + "loud": 5189, + "thousands": 5190, + "worried": 5191, + "shadow": 5192, + "transportation": 5193, + "horses": 5194, + "weapon": 5195, + "arena": 5196, + "importance": 5197, + "users": 5198, + "tim": 5199, + "objects": 5200, + "contributed": 5201, + "dragon": 5202, + "douglas": 5203, + "aware": 5204, + "senator": 5205, + "johnny": 5206, + "jordan": 5207, + "sisters": 5208, + "engines": 5209, + "flag": 5210, + "investment": 5211, + "samuel": 5212, + "shock": 5213, + "capable": 5214, + "clark": 5215, + "row": 5216, + "wheel": 5217, + "refers": 5218, + "session": 5219, + "familiar": 5220, + "biggest": 5221, + "wins": 5222, + "hate": 5223, + "maintained": 5224, + "drove": 5225, + "hamilton": 5226, + "request": 5227, + "expressed": 5228, + "injured": 5229, + "underground": 5230, + "churches": 5231, + "walker": 5232, + "wars": 5233, + "tunnel": 5234, + "passes": 5235, + "stupid": 5236, + "agriculture": 5237, + "softly": 5238, + "cabinet": 5239, + "regarded": 5240, + "joining": 5241, + "indiana": 5242, + "##ea": 5243, + "##ms": 5244, + "push": 5245, + "dates": 5246, + "spend": 5247, + "behavior": 5248, + "woods": 5249, + "protein": 5250, + "gently": 5251, + "chase": 5252, + "morgan": 5253, + "mention": 5254, + "burning": 5255, + "wake": 5256, + "combination": 5257, + "occur": 5258, + "mirror": 5259, + "leads": 5260, + "jimmy": 5261, + "indeed": 5262, + "impossible": 5263, + "singapore": 5264, + "paintings": 5265, + "covering": 5266, + "##nes": 5267, + "soldier": 5268, + "locations": 5269, + "attendance": 5270, + "sell": 5271, + "historian": 5272, + "wisconsin": 5273, + "invasion": 5274, + "argued": 5275, + "painter": 5276, + "diego": 5277, + "changing": 5278, + "egypt": 5279, + "##don": 5280, + "experienced": 5281, + "inches": 5282, + "##ku": 5283, + "missouri": 5284, + "vol": 5285, + "grounds": 5286, + "spoken": 5287, + "switzerland": 5288, + "##gan": 5289, + "reform": 5290, + "rolling": 5291, + "ha": 5292, + "forget": 5293, + "massive": 5294, + "resigned": 5295, + "burned": 5296, + "allen": 5297, + "tennessee": 5298, + "locked": 5299, + "values": 5300, + "improved": 5301, + "##mo": 5302, + "wounded": 5303, + "universe": 5304, + "sick": 5305, + "dating": 5306, + "facing": 5307, + "pack": 5308, + "purchase": 5309, + "user": 5310, + "##pur": 5311, + "moments": 5312, + "##ul": 5313, + "merged": 5314, + "anniversary": 5315, + "1908": 5316, + "coal": 5317, + "brick": 5318, + "understood": 5319, + "causes": 5320, + "dynasty": 5321, + "queensland": 5322, + "establish": 5323, + "stores": 5324, + "crisis": 5325, + "promote": 5326, + "hoping": 5327, + "views": 5328, + "cards": 5329, + "referee": 5330, + "extension": 5331, + "##si": 5332, + "raise": 5333, + "arizona": 5334, + "improve": 5335, + "colonial": 5336, + "formal": 5337, + "charged": 5338, + "##rt": 5339, + "palm": 5340, + "lucky": 5341, + "hide": 5342, + "rescue": 5343, + "faces": 5344, + "95": 5345, + "feelings": 5346, + "candidates": 5347, + "juan": 5348, + "##ell": 5349, + "goods": 5350, + "6th": 5351, + "courses": 5352, + "weekend": 5353, + "59": 5354, + "luke": 5355, + "cash": 5356, + "fallen": 5357, + "##om": 5358, + "delivered": 5359, + "affected": 5360, + "installed": 5361, + "carefully": 5362, + "tries": 5363, + "swiss": 5364, + "hollywood": 5365, + "costs": 5366, + "lincoln": 5367, + "responsibility": 5368, + "##he": 5369, + "shore": 5370, + "file": 5371, + "proper": 5372, + "normally": 5373, + "maryland": 5374, + "assistance": 5375, + "jump": 5376, + "constant": 5377, + "offering": 5378, + "friendly": 5379, + "waters": 5380, + "persons": 5381, + "realize": 5382, + "contain": 5383, + "trophy": 5384, + "800": 5385, + "partnership": 5386, + "factor": 5387, + "58": 5388, + "musicians": 5389, + "cry": 5390, + "bound": 5391, + "oregon": 5392, + "indicated": 5393, + "hero": 5394, + "houston": 5395, + "medium": 5396, + "##ure": 5397, + "consisting": 5398, + "somewhat": 5399, + "##ara": 5400, + "57": 5401, + "cycle": 5402, + "##che": 5403, + "beer": 5404, + "moore": 5405, + "frederick": 5406, + "gotten": 5407, + "eleven": 5408, + "worst": 5409, + "weak": 5410, + "approached": 5411, + "arranged": 5412, + "chin": 5413, + "loan": 5414, + "universal": 5415, + "bond": 5416, + "fifteen": 5417, + "pattern": 5418, + "disappeared": 5419, + "##ney": 5420, + "translated": 5421, + "##zed": 5422, + "lip": 5423, + "arab": 5424, + "capture": 5425, + "interests": 5426, + "insurance": 5427, + "##chi": 5428, + "shifted": 5429, + "cave": 5430, + "prix": 5431, + "warning": 5432, + "sections": 5433, + "courts": 5434, + "coat": 5435, + "plot": 5436, + "smell": 5437, + "feed": 5438, + "golf": 5439, + "favorite": 5440, + "maintain": 5441, + "knife": 5442, + "vs": 5443, + "voted": 5444, + "degrees": 5445, + "finance": 5446, + "quebec": 5447, + "opinion": 5448, + "translation": 5449, + "manner": 5450, + "ruled": 5451, + "operate": 5452, + "productions": 5453, + "choose": 5454, + "musician": 5455, + "discovery": 5456, + "confused": 5457, + "tired": 5458, + "separated": 5459, + "stream": 5460, + "techniques": 5461, + "committed": 5462, + "attend": 5463, + "ranking": 5464, + "kings": 5465, + "throw": 5466, + "passengers": 5467, + "measure": 5468, + "horror": 5469, + "fan": 5470, + "mining": 5471, + "sand": 5472, + "danger": 5473, + "salt": 5474, + "calm": 5475, + "decade": 5476, + "dam": 5477, + "require": 5478, + "runner": 5479, + "##ik": 5480, + "rush": 5481, + "associate": 5482, + "greece": 5483, + "##ker": 5484, + "rivers": 5485, + "consecutive": 5486, + "matthew": 5487, + "##ski": 5488, + "sighed": 5489, + "sq": 5490, + "documents": 5491, + "steam": 5492, + "edited": 5493, + "closing": 5494, + "tie": 5495, + "accused": 5496, + "1905": 5497, + "##ini": 5498, + "islamic": 5499, + "distributed": 5500, + "directors": 5501, + "organisation": 5502, + "bruce": 5503, + "7th": 5504, + "breathing": 5505, + "mad": 5506, + "lit": 5507, + "arrival": 5508, + "concrete": 5509, + "taste": 5510, + "08": 5511, + "composition": 5512, + "shaking": 5513, + "faster": 5514, + "amateur": 5515, + "adjacent": 5516, + "stating": 5517, + "1906": 5518, + "twin": 5519, + "flew": 5520, + "##ran": 5521, + "tokyo": 5522, + "publications": 5523, + "##tone": 5524, + "obviously": 5525, + "ridge": 5526, + "storage": 5527, + "1907": 5528, + "carl": 5529, + "pages": 5530, + "concluded": 5531, + "desert": 5532, + "driven": 5533, + "universities": 5534, + "ages": 5535, + "terminal": 5536, + "sequence": 5537, + "borough": 5538, + "250": 5539, + "constituency": 5540, + "creative": 5541, + "cousin": 5542, + "economics": 5543, + "dreams": 5544, + "margaret": 5545, + "notably": 5546, + "reduce": 5547, + "montreal": 5548, + "mode": 5549, + "17th": 5550, + "ears": 5551, + "saved": 5552, + "jan": 5553, + "vocal": 5554, + "##ica": 5555, + "1909": 5556, + "andy": 5557, + "##jo": 5558, + "riding": 5559, + "roughly": 5560, + "threatened": 5561, + "##ise": 5562, + "meters": 5563, + "meanwhile": 5564, + "landed": 5565, + "compete": 5566, + "repeated": 5567, + "grass": 5568, + "czech": 5569, + "regularly": 5570, + "charges": 5571, + "tea": 5572, + "sudden": 5573, + "appeal": 5574, + "##ung": 5575, + "solution": 5576, + "describes": 5577, + "pierre": 5578, + "classification": 5579, + "glad": 5580, + "parking": 5581, + "##ning": 5582, + "belt": 5583, + "physics": 5584, + "99": 5585, + "rachel": 5586, + "add": 5587, + "hungarian": 5588, + "participate": 5589, + "expedition": 5590, + "damaged": 5591, + "gift": 5592, + "childhood": 5593, + "85": 5594, + "fifty": 5595, + "##red": 5596, + "mathematics": 5597, + "jumped": 5598, + "letting": 5599, + "defensive": 5600, + "mph": 5601, + "##ux": 5602, + "##gh": 5603, + "testing": 5604, + "##hip": 5605, + "hundreds": 5606, + "shoot": 5607, + "owners": 5608, + "matters": 5609, + "smoke": 5610, + "israeli": 5611, + "kentucky": 5612, + "dancing": 5613, + "mounted": 5614, + "grandfather": 5615, + "emma": 5616, + "designs": 5617, + "profit": 5618, + "argentina": 5619, + "##gs": 5620, + "truly": 5621, + "li": 5622, + "lawrence": 5623, + "cole": 5624, + "begun": 5625, + "detroit": 5626, + "willing": 5627, + "branches": 5628, + "smiling": 5629, + "decide": 5630, + "miami": 5631, + "enjoyed": 5632, + "recordings": 5633, + "##dale": 5634, + "poverty": 5635, + "ethnic": 5636, + "gay": 5637, + "##bi": 5638, + "gary": 5639, + "arabic": 5640, + "09": 5641, + "accompanied": 5642, + "##one": 5643, + "##ons": 5644, + "fishing": 5645, + "determine": 5646, + "residential": 5647, + "acid": 5648, + "##ary": 5649, + "alice": 5650, + "returns": 5651, + "starred": 5652, + "mail": 5653, + "##ang": 5654, + "jonathan": 5655, + "strategy": 5656, + "##ue": 5657, + "net": 5658, + "forty": 5659, + "cook": 5660, + "businesses": 5661, + "equivalent": 5662, + "commonwealth": 5663, + "distinct": 5664, + "ill": 5665, + "##cy": 5666, + "seriously": 5667, + "##ors": 5668, + "##ped": 5669, + "shift": 5670, + "harris": 5671, + "replace": 5672, + "rio": 5673, + "imagine": 5674, + "formula": 5675, + "ensure": 5676, + "##ber": 5677, + "additionally": 5678, + "scheme": 5679, + "conservation": 5680, + "occasionally": 5681, + "purposes": 5682, + "feels": 5683, + "favor": 5684, + "##and": 5685, + "##ore": 5686, + "1930s": 5687, + "contrast": 5688, + "hanging": 5689, + "hunt": 5690, + "movies": 5691, + "1904": 5692, + "instruments": 5693, + "victims": 5694, + "danish": 5695, + "christopher": 5696, + "busy": 5697, + "demon": 5698, + "sugar": 5699, + "earliest": 5700, + "colony": 5701, + "studying": 5702, + "balance": 5703, + "duties": 5704, + "##ks": 5705, + "belgium": 5706, + "slipped": 5707, + "carter": 5708, + "05": 5709, + "visible": 5710, + "stages": 5711, + "iraq": 5712, + "fifa": 5713, + "##im": 5714, + "commune": 5715, + "forming": 5716, + "zero": 5717, + "07": 5718, + "continuing": 5719, + "talked": 5720, + "counties": 5721, + "legend": 5722, + "bathroom": 5723, + "option": 5724, + "tail": 5725, + "clay": 5726, + "daughters": 5727, + "afterwards": 5728, + "severe": 5729, + "jaw": 5730, + "visitors": 5731, + "##ded": 5732, + "devices": 5733, + "aviation": 5734, + "russell": 5735, + "kate": 5736, + "##vi": 5737, + "entering": 5738, + "subjects": 5739, + "##ino": 5740, + "temporary": 5741, + "swimming": 5742, + "forth": 5743, + "smooth": 5744, + "ghost": 5745, + "audio": 5746, + "bush": 5747, + "operates": 5748, + "rocks": 5749, + "movements": 5750, + "signs": 5751, + "eddie": 5752, + "##tz": 5753, + "ann": 5754, + "voices": 5755, + "honorary": 5756, + "06": 5757, + "memories": 5758, + "dallas": 5759, + "pure": 5760, + "measures": 5761, + "racial": 5762, + "promised": 5763, + "66": 5764, + "harvard": 5765, + "ceo": 5766, + "16th": 5767, + "parliamentary": 5768, + "indicate": 5769, + "benefit": 5770, + "flesh": 5771, + "dublin": 5772, + "louisiana": 5773, + "1902": 5774, + "1901": 5775, + "patient": 5776, + "sleeping": 5777, + "1903": 5778, + "membership": 5779, + "coastal": 5780, + "medieval": 5781, + "wanting": 5782, + "element": 5783, + "scholars": 5784, + "rice": 5785, + "62": 5786, + "limit": 5787, + "survive": 5788, + "makeup": 5789, + "rating": 5790, + "definitely": 5791, + "collaboration": 5792, + "obvious": 5793, + "##tan": 5794, + "boss": 5795, + "ms": 5796, + "baron": 5797, + "birthday": 5798, + "linked": 5799, + "soil": 5800, + "diocese": 5801, + "##lan": 5802, + "ncaa": 5803, + "##mann": 5804, + "offensive": 5805, + "shell": 5806, + "shouldn": 5807, + "waist": 5808, + "##tus": 5809, + "plain": 5810, + "ross": 5811, + "organ": 5812, + "resolution": 5813, + "manufacturing": 5814, + "adding": 5815, + "relative": 5816, + "kennedy": 5817, + "98": 5818, + "whilst": 5819, + "moth": 5820, + "marketing": 5821, + "gardens": 5822, + "crash": 5823, + "72": 5824, + "heading": 5825, + "partners": 5826, + "credited": 5827, + "carlos": 5828, + "moves": 5829, + "cable": 5830, + "##zi": 5831, + "marshall": 5832, + "##out": 5833, + "depending": 5834, + "bottle": 5835, + "represents": 5836, + "rejected": 5837, + "responded": 5838, + "existed": 5839, + "04": 5840, + "jobs": 5841, + "denmark": 5842, + "lock": 5843, + "##ating": 5844, + "treated": 5845, + "graham": 5846, + "routes": 5847, + "talent": 5848, + "commissioner": 5849, + "drugs": 5850, + "secure": 5851, + "tests": 5852, + "reign": 5853, + "restored": 5854, + "photography": 5855, + "##gi": 5856, + "contributions": 5857, + "oklahoma": 5858, + "designer": 5859, + "disc": 5860, + "grin": 5861, + "seattle": 5862, + "robin": 5863, + "paused": 5864, + "atlanta": 5865, + "unusual": 5866, + "##gate": 5867, + "praised": 5868, + "las": 5869, + "laughing": 5870, + "satellite": 5871, + "hungary": 5872, + "visiting": 5873, + "##sky": 5874, + "interesting": 5875, + "factors": 5876, + "deck": 5877, + "poems": 5878, + "norman": 5879, + "##water": 5880, + "stuck": 5881, + "speaker": 5882, + "rifle": 5883, + "domain": 5884, + "premiered": 5885, + "##her": 5886, + "dc": 5887, + "comics": 5888, + "actors": 5889, + "01": 5890, + "reputation": 5891, + "eliminated": 5892, + "8th": 5893, + "ceiling": 5894, + "prisoners": 5895, + "script": 5896, + "##nce": 5897, + "leather": 5898, + "austin": 5899, + "mississippi": 5900, + "rapidly": 5901, + "admiral": 5902, + "parallel": 5903, + "charlotte": 5904, + "guilty": 5905, + "tools": 5906, + "gender": 5907, + "divisions": 5908, + "fruit": 5909, + "##bs": 5910, + "laboratory": 5911, + "nelson": 5912, + "fantasy": 5913, + "marry": 5914, + "rapid": 5915, + "aunt": 5916, + "tribe": 5917, + "requirements": 5918, + "aspects": 5919, + "suicide": 5920, + "amongst": 5921, + "adams": 5922, + "bone": 5923, + "ukraine": 5924, + "abc": 5925, + "kick": 5926, + "sees": 5927, + "edinburgh": 5928, + "clothing": 5929, + "column": 5930, + "rough": 5931, + "gods": 5932, + "hunting": 5933, + "broadway": 5934, + "gathered": 5935, + "concerns": 5936, + "##ek": 5937, + "spending": 5938, + "ty": 5939, + "12th": 5940, + "snapped": 5941, + "requires": 5942, + "solar": 5943, + "bones": 5944, + "cavalry": 5945, + "##tta": 5946, + "iowa": 5947, + "drinking": 5948, + "waste": 5949, + "index": 5950, + "franklin": 5951, + "charity": 5952, + "thompson": 5953, + "stewart": 5954, + "tip": 5955, + "flash": 5956, + "landscape": 5957, + "friday": 5958, + "enjoy": 5959, + "singh": 5960, + "poem": 5961, + "listening": 5962, + "##back": 5963, + "eighth": 5964, + "fred": 5965, + "differences": 5966, + "adapted": 5967, + "bomb": 5968, + "ukrainian": 5969, + "surgery": 5970, + "corporate": 5971, + "masters": 5972, + "anywhere": 5973, + "##more": 5974, + "waves": 5975, + "odd": 5976, + "sean": 5977, + "portugal": 5978, + "orleans": 5979, + "dick": 5980, + "debate": 5981, + "kent": 5982, + "eating": 5983, + "puerto": 5984, + "cleared": 5985, + "96": 5986, + "expect": 5987, + "cinema": 5988, + "97": 5989, + "guitarist": 5990, + "blocks": 5991, + "electrical": 5992, + "agree": 5993, + "involving": 5994, + "depth": 5995, + "dying": 5996, + "panel": 5997, + "struggle": 5998, + "##ged": 5999, + "peninsula": 6000, + "adults": 6001, + "novels": 6002, + "emerged": 6003, + "vienna": 6004, + "metro": 6005, + "debuted": 6006, + "shoes": 6007, + "tamil": 6008, + "songwriter": 6009, + "meets": 6010, + "prove": 6011, + "beating": 6012, + "instance": 6013, + "heaven": 6014, + "scared": 6015, + "sending": 6016, + "marks": 6017, + "artistic": 6018, + "passage": 6019, + "superior": 6020, + "03": 6021, + "significantly": 6022, + "shopping": 6023, + "##tive": 6024, + "retained": 6025, + "##izing": 6026, + "malaysia": 6027, + "technique": 6028, + "cheeks": 6029, + "##ola": 6030, + "warren": 6031, + "maintenance": 6032, + "destroy": 6033, + "extreme": 6034, + "allied": 6035, + "120": 6036, + "appearing": 6037, + "##yn": 6038, + "fill": 6039, + "advice": 6040, + "alabama": 6041, + "qualifying": 6042, + "policies": 6043, + "cleveland": 6044, + "hat": 6045, + "battery": 6046, + "smart": 6047, + "authors": 6048, + "10th": 6049, + "soundtrack": 6050, + "acted": 6051, + "dated": 6052, + "lb": 6053, + "glance": 6054, + "equipped": 6055, + "coalition": 6056, + "funny": 6057, + "outer": 6058, + "ambassador": 6059, + "roy": 6060, + "possibility": 6061, + "couples": 6062, + "campbell": 6063, + "dna": 6064, + "loose": 6065, + "ethan": 6066, + "supplies": 6067, + "1898": 6068, + "gonna": 6069, + "88": 6070, + "monster": 6071, + "##res": 6072, + "shake": 6073, + "agents": 6074, + "frequency": 6075, + "springs": 6076, + "dogs": 6077, + "practices": 6078, + "61": 6079, + "gang": 6080, + "plastic": 6081, + "easier": 6082, + "suggests": 6083, + "gulf": 6084, + "blade": 6085, + "exposed": 6086, + "colors": 6087, + "industries": 6088, + "markets": 6089, + "pan": 6090, + "nervous": 6091, + "electoral": 6092, + "charts": 6093, + "legislation": 6094, + "ownership": 6095, + "##idae": 6096, + "mac": 6097, + "appointment": 6098, + "shield": 6099, + "copy": 6100, + "assault": 6101, + "socialist": 6102, + "abbey": 6103, + "monument": 6104, + "license": 6105, + "throne": 6106, + "employment": 6107, + "jay": 6108, + "93": 6109, + "replacement": 6110, + "charter": 6111, + "cloud": 6112, + "powered": 6113, + "suffering": 6114, + "accounts": 6115, + "oak": 6116, + "connecticut": 6117, + "strongly": 6118, + "wright": 6119, + "colour": 6120, + "crystal": 6121, + "13th": 6122, + "context": 6123, + "welsh": 6124, + "networks": 6125, + "voiced": 6126, + "gabriel": 6127, + "jerry": 6128, + "##cing": 6129, + "forehead": 6130, + "mp": 6131, + "##ens": 6132, + "manage": 6133, + "schedule": 6134, + "totally": 6135, + "remix": 6136, + "##ii": 6137, + "forests": 6138, + "occupation": 6139, + "print": 6140, + "nicholas": 6141, + "brazilian": 6142, + "strategic": 6143, + "vampires": 6144, + "engineers": 6145, + "76": 6146, + "roots": 6147, + "seek": 6148, + "correct": 6149, + "instrumental": 6150, + "und": 6151, + "alfred": 6152, + "backed": 6153, + "hop": 6154, + "##des": 6155, + "stanley": 6156, + "robinson": 6157, + "traveled": 6158, + "wayne": 6159, + "welcome": 6160, + "austrian": 6161, + "achieve": 6162, + "67": 6163, + "exit": 6164, + "rates": 6165, + "1899": 6166, + "strip": 6167, + "whereas": 6168, + "##cs": 6169, + "sing": 6170, + "deeply": 6171, + "adventure": 6172, + "bobby": 6173, + "rick": 6174, + "jamie": 6175, + "careful": 6176, + "components": 6177, + "cap": 6178, + "useful": 6179, + "personality": 6180, + "knee": 6181, + "##shi": 6182, + "pushing": 6183, + "hosts": 6184, + "02": 6185, + "protest": 6186, + "ca": 6187, + "ottoman": 6188, + "symphony": 6189, + "##sis": 6190, + "63": 6191, + "boundary": 6192, + "1890": 6193, + "processes": 6194, + "considering": 6195, + "considerable": 6196, + "tons": 6197, + "##work": 6198, + "##ft": 6199, + "##nia": 6200, + "cooper": 6201, + "trading": 6202, + "dear": 6203, + "conduct": 6204, + "91": 6205, + "illegal": 6206, + "apple": 6207, + "revolutionary": 6208, + "holiday": 6209, + "definition": 6210, + "harder": 6211, + "##van": 6212, + "jacob": 6213, + "circumstances": 6214, + "destruction": 6215, + "##lle": 6216, + "popularity": 6217, + "grip": 6218, + "classified": 6219, + "liverpool": 6220, + "donald": 6221, + "baltimore": 6222, + "flows": 6223, + "seeking": 6224, + "honour": 6225, + "approval": 6226, + "92": 6227, + "mechanical": 6228, + "till": 6229, + "happening": 6230, + "statue": 6231, + "critic": 6232, + "increasingly": 6233, + "immediate": 6234, + "describe": 6235, + "commerce": 6236, + "stare": 6237, + "##ster": 6238, + "indonesia": 6239, + "meat": 6240, + "rounds": 6241, + "boats": 6242, + "baker": 6243, + "orthodox": 6244, + "depression": 6245, + "formally": 6246, + "worn": 6247, + "naked": 6248, + "claire": 6249, + "muttered": 6250, + "sentence": 6251, + "11th": 6252, + "emily": 6253, + "document": 6254, + "77": 6255, + "criticism": 6256, + "wished": 6257, + "vessel": 6258, + "spiritual": 6259, + "bent": 6260, + "virgin": 6261, + "parker": 6262, + "minimum": 6263, + "murray": 6264, + "lunch": 6265, + "danny": 6266, + "printed": 6267, + "compilation": 6268, + "keyboards": 6269, + "false": 6270, + "blow": 6271, + "belonged": 6272, + "68": 6273, + "raising": 6274, + "78": 6275, + "cutting": 6276, + "##board": 6277, + "pittsburgh": 6278, + "##up": 6279, + "9th": 6280, + "shadows": 6281, + "81": 6282, + "hated": 6283, + "indigenous": 6284, + "jon": 6285, + "15th": 6286, + "barry": 6287, + "scholar": 6288, + "ah": 6289, + "##zer": 6290, + "oliver": 6291, + "##gy": 6292, + "stick": 6293, + "susan": 6294, + "meetings": 6295, + "attracted": 6296, + "spell": 6297, + "romantic": 6298, + "##ver": 6299, + "ye": 6300, + "1895": 6301, + "photo": 6302, + "demanded": 6303, + "customers": 6304, + "##ac": 6305, + "1896": 6306, + "logan": 6307, + "revival": 6308, + "keys": 6309, + "modified": 6310, + "commanded": 6311, + "jeans": 6312, + "##ious": 6313, + "upset": 6314, + "raw": 6315, + "phil": 6316, + "detective": 6317, + "hiding": 6318, + "resident": 6319, + "vincent": 6320, + "##bly": 6321, + "experiences": 6322, + "diamond": 6323, + "defeating": 6324, + "coverage": 6325, + "lucas": 6326, + "external": 6327, + "parks": 6328, + "franchise": 6329, + "helen": 6330, + "bible": 6331, + "successor": 6332, + "percussion": 6333, + "celebrated": 6334, + "il": 6335, + "lift": 6336, + "profile": 6337, + "clan": 6338, + "romania": 6339, + "##ied": 6340, + "mills": 6341, + "##su": 6342, + "nobody": 6343, + "achievement": 6344, + "shrugged": 6345, + "fault": 6346, + "1897": 6347, + "rhythm": 6348, + "initiative": 6349, + "breakfast": 6350, + "carbon": 6351, + "700": 6352, + "69": 6353, + "lasted": 6354, + "violent": 6355, + "74": 6356, + "wound": 6357, + "ken": 6358, + "killer": 6359, + "gradually": 6360, + "filmed": 6361, + "°c": 6362, + "dollars": 6363, + "processing": 6364, + "94": 6365, + "remove": 6366, + "criticized": 6367, + "guests": 6368, + "sang": 6369, + "chemistry": 6370, + "##vin": 6371, + "legislature": 6372, + "disney": 6373, + "##bridge": 6374, + "uniform": 6375, + "escaped": 6376, + "integrated": 6377, + "proposal": 6378, + "purple": 6379, + "denied": 6380, + "liquid": 6381, + "karl": 6382, + "influential": 6383, + "morris": 6384, + "nights": 6385, + "stones": 6386, + "intense": 6387, + "experimental": 6388, + "twisted": 6389, + "71": 6390, + "84": 6391, + "##ld": 6392, + "pace": 6393, + "nazi": 6394, + "mitchell": 6395, + "ny": 6396, + "blind": 6397, + "reporter": 6398, + "newspapers": 6399, + "14th": 6400, + "centers": 6401, + "burn": 6402, + "basin": 6403, + "forgotten": 6404, + "surviving": 6405, + "filed": 6406, + "collections": 6407, + "monastery": 6408, + "losses": 6409, + "manual": 6410, + "couch": 6411, + "description": 6412, + "appropriate": 6413, + "merely": 6414, + "tag": 6415, + "missions": 6416, + "sebastian": 6417, + "restoration": 6418, + "replacing": 6419, + "triple": 6420, + "73": 6421, + "elder": 6422, + "julia": 6423, + "warriors": 6424, + "benjamin": 6425, + "julian": 6426, + "convinced": 6427, + "stronger": 6428, + "amazing": 6429, + "declined": 6430, + "versus": 6431, + "merchant": 6432, + "happens": 6433, + "output": 6434, + "finland": 6435, + "bare": 6436, + "barbara": 6437, + "absence": 6438, + "ignored": 6439, + "dawn": 6440, + "injuries": 6441, + "##port": 6442, + "producers": 6443, + "##ram": 6444, + "82": 6445, + "luis": 6446, + "##ities": 6447, + "kw": 6448, + "admit": 6449, + "expensive": 6450, + "electricity": 6451, + "nba": 6452, + "exception": 6453, + "symbol": 6454, + "##ving": 6455, + "ladies": 6456, + "shower": 6457, + "sheriff": 6458, + "characteristics": 6459, + "##je": 6460, + "aimed": 6461, + "button": 6462, + "ratio": 6463, + "effectively": 6464, + "summit": 6465, + "angle": 6466, + "jury": 6467, + "bears": 6468, + "foster": 6469, + "vessels": 6470, + "pants": 6471, + "executed": 6472, + "evans": 6473, + "dozen": 6474, + "advertising": 6475, + "kicked": 6476, + "patrol": 6477, + "1889": 6478, + "competitions": 6479, + "lifetime": 6480, + "principles": 6481, + "athletics": 6482, + "##logy": 6483, + "birmingham": 6484, + "sponsored": 6485, + "89": 6486, + "rob": 6487, + "nomination": 6488, + "1893": 6489, + "acoustic": 6490, + "##sm": 6491, + "creature": 6492, + "longest": 6493, + "##tra": 6494, + "credits": 6495, + "harbor": 6496, + "dust": 6497, + "josh": 6498, + "##so": 6499, + "territories": 6500, + "milk": 6501, + "infrastructure": 6502, + "completion": 6503, + "thailand": 6504, + "indians": 6505, + "leon": 6506, + "archbishop": 6507, + "##sy": 6508, + "assist": 6509, + "pitch": 6510, + "blake": 6511, + "arrangement": 6512, + "girlfriend": 6513, + "serbian": 6514, + "operational": 6515, + "hence": 6516, + "sad": 6517, + "scent": 6518, + "fur": 6519, + "dj": 6520, + "sessions": 6521, + "hp": 6522, + "refer": 6523, + "rarely": 6524, + "##ora": 6525, + "exists": 6526, + "1892": 6527, + "##ten": 6528, + "scientists": 6529, + "dirty": 6530, + "penalty": 6531, + "burst": 6532, + "portrait": 6533, + "seed": 6534, + "79": 6535, + "pole": 6536, + "limits": 6537, + "rival": 6538, + "1894": 6539, + "stable": 6540, + "alpha": 6541, + "grave": 6542, + "constitutional": 6543, + "alcohol": 6544, + "arrest": 6545, + "flower": 6546, + "mystery": 6547, + "devil": 6548, + "architectural": 6549, + "relationships": 6550, + "greatly": 6551, + "habitat": 6552, + "##istic": 6553, + "larry": 6554, + "progressive": 6555, + "remote": 6556, + "cotton": 6557, + "##ics": 6558, + "##ok": 6559, + "preserved": 6560, + "reaches": 6561, + "##ming": 6562, + "cited": 6563, + "86": 6564, + "vast": 6565, + "scholarship": 6566, + "decisions": 6567, + "cbs": 6568, + "joy": 6569, + "teach": 6570, + "1885": 6571, + "editions": 6572, + "knocked": 6573, + "eve": 6574, + "searching": 6575, + "partly": 6576, + "participation": 6577, + "gap": 6578, + "animated": 6579, + "fate": 6580, + "excellent": 6581, + "##ett": 6582, + "na": 6583, + "87": 6584, + "alternate": 6585, + "saints": 6586, + "youngest": 6587, + "##ily": 6588, + "climbed": 6589, + "##ita": 6590, + "##tors": 6591, + "suggest": 6592, + "##ct": 6593, + "discussion": 6594, + "staying": 6595, + "choir": 6596, + "lakes": 6597, + "jacket": 6598, + "revenue": 6599, + "nevertheless": 6600, + "peaked": 6601, + "instrument": 6602, + "wondering": 6603, + "annually": 6604, + "managing": 6605, + "neil": 6606, + "1891": 6607, + "signing": 6608, + "terry": 6609, + "##ice": 6610, + "apply": 6611, + "clinical": 6612, + "brooklyn": 6613, + "aim": 6614, + "catherine": 6615, + "fuck": 6616, + "farmers": 6617, + "figured": 6618, + "ninth": 6619, + "pride": 6620, + "hugh": 6621, + "evolution": 6622, + "ordinary": 6623, + "involvement": 6624, + "comfortable": 6625, + "shouted": 6626, + "tech": 6627, + "encouraged": 6628, + "taiwan": 6629, + "representation": 6630, + "sharing": 6631, + "##lia": 6632, + "##em": 6633, + "panic": 6634, + "exact": 6635, + "cargo": 6636, + "competing": 6637, + "fat": 6638, + "cried": 6639, + "83": 6640, + "1920s": 6641, + "occasions": 6642, + "pa": 6643, + "cabin": 6644, + "borders": 6645, + "utah": 6646, + "marcus": 6647, + "##isation": 6648, + "badly": 6649, + "muscles": 6650, + "##ance": 6651, + "victorian": 6652, + "transition": 6653, + "warner": 6654, + "bet": 6655, + "permission": 6656, + "##rin": 6657, + "slave": 6658, + "terrible": 6659, + "similarly": 6660, + "shares": 6661, + "seth": 6662, + "uefa": 6663, + "possession": 6664, + "medals": 6665, + "benefits": 6666, + "colleges": 6667, + "lowered": 6668, + "perfectly": 6669, + "mall": 6670, + "transit": 6671, + "##ye": 6672, + "##kar": 6673, + "publisher": 6674, + "##ened": 6675, + "harrison": 6676, + "deaths": 6677, + "elevation": 6678, + "##ae": 6679, + "asleep": 6680, + "machines": 6681, + "sigh": 6682, + "ash": 6683, + "hardly": 6684, + "argument": 6685, + "occasion": 6686, + "parent": 6687, + "leo": 6688, + "decline": 6689, + "1888": 6690, + "contribution": 6691, + "##ua": 6692, + "concentration": 6693, + "1000": 6694, + "opportunities": 6695, + "hispanic": 6696, + "guardian": 6697, + "extent": 6698, + "emotions": 6699, + "hips": 6700, + "mason": 6701, + "volumes": 6702, + "bloody": 6703, + "controversy": 6704, + "diameter": 6705, + "steady": 6706, + "mistake": 6707, + "phoenix": 6708, + "identify": 6709, + "violin": 6710, + "##sk": 6711, + "departure": 6712, + "richmond": 6713, + "spin": 6714, + "funeral": 6715, + "enemies": 6716, + "1864": 6717, + "gear": 6718, + "literally": 6719, + "connor": 6720, + "random": 6721, + "sergeant": 6722, + "grab": 6723, + "confusion": 6724, + "1865": 6725, + "transmission": 6726, + "informed": 6727, + "op": 6728, + "leaning": 6729, + "sacred": 6730, + "suspended": 6731, + "thinks": 6732, + "gates": 6733, + "portland": 6734, + "luck": 6735, + "agencies": 6736, + "yours": 6737, + "hull": 6738, + "expert": 6739, + "muscle": 6740, + "layer": 6741, + "practical": 6742, + "sculpture": 6743, + "jerusalem": 6744, + "latest": 6745, + "lloyd": 6746, + "statistics": 6747, + "deeper": 6748, + "recommended": 6749, + "warrior": 6750, + "arkansas": 6751, + "mess": 6752, + "supports": 6753, + "greg": 6754, + "eagle": 6755, + "1880": 6756, + "recovered": 6757, + "rated": 6758, + "concerts": 6759, + "rushed": 6760, + "##ano": 6761, + "stops": 6762, + "eggs": 6763, + "files": 6764, + "premiere": 6765, + "keith": 6766, + "##vo": 6767, + "delhi": 6768, + "turner": 6769, + "pit": 6770, + "affair": 6771, + "belief": 6772, + "paint": 6773, + "##zing": 6774, + "mate": 6775, + "##ach": 6776, + "##ev": 6777, + "victim": 6778, + "##ology": 6779, + "withdrew": 6780, + "bonus": 6781, + "styles": 6782, + "fled": 6783, + "##ud": 6784, + "glasgow": 6785, + "technologies": 6786, + "funded": 6787, + "nbc": 6788, + "adaptation": 6789, + "##ata": 6790, + "portrayed": 6791, + "cooperation": 6792, + "supporters": 6793, + "judges": 6794, + "bernard": 6795, + "justin": 6796, + "hallway": 6797, + "ralph": 6798, + "##ick": 6799, + "graduating": 6800, + "controversial": 6801, + "distant": 6802, + "continental": 6803, + "spider": 6804, + "bite": 6805, + "##ho": 6806, + "recognize": 6807, + "intention": 6808, + "mixing": 6809, + "##ese": 6810, + "egyptian": 6811, + "bow": 6812, + "tourism": 6813, + "suppose": 6814, + "claiming": 6815, + "tiger": 6816, + "dominated": 6817, + "participants": 6818, + "vi": 6819, + "##ru": 6820, + "nurse": 6821, + "partially": 6822, + "tape": 6823, + "##rum": 6824, + "psychology": 6825, + "##rn": 6826, + "essential": 6827, + "touring": 6828, + "duo": 6829, + "voting": 6830, + "civilian": 6831, + "emotional": 6832, + "channels": 6833, + "##king": 6834, + "apparent": 6835, + "hebrew": 6836, + "1887": 6837, + "tommy": 6838, + "carrier": 6839, + "intersection": 6840, + "beast": 6841, + "hudson": 6842, + "##gar": 6843, + "##zo": 6844, + "lab": 6845, + "nova": 6846, + "bench": 6847, + "discuss": 6848, + "costa": 6849, + "##ered": 6850, + "detailed": 6851, + "behalf": 6852, + "drivers": 6853, + "unfortunately": 6854, + "obtain": 6855, + "##lis": 6856, + "rocky": 6857, + "##dae": 6858, + "siege": 6859, + "friendship": 6860, + "honey": 6861, + "##rian": 6862, + "1861": 6863, + "amy": 6864, + "hang": 6865, + "posted": 6866, + "governments": 6867, + "collins": 6868, + "respond": 6869, + "wildlife": 6870, + "preferred": 6871, + "operator": 6872, + "##po": 6873, + "laura": 6874, + "pregnant": 6875, + "videos": 6876, + "dennis": 6877, + "suspected": 6878, + "boots": 6879, + "instantly": 6880, + "weird": 6881, + "automatic": 6882, + "businessman": 6883, + "alleged": 6884, + "placing": 6885, + "throwing": 6886, + "ph": 6887, + "mood": 6888, + "1862": 6889, + "perry": 6890, + "venue": 6891, + "jet": 6892, + "remainder": 6893, + "##lli": 6894, + "##ci": 6895, + "passion": 6896, + "biological": 6897, + "boyfriend": 6898, + "1863": 6899, + "dirt": 6900, + "buffalo": 6901, + "ron": 6902, + "segment": 6903, + "fa": 6904, + "abuse": 6905, + "##era": 6906, + "genre": 6907, + "thrown": 6908, + "stroke": 6909, + "colored": 6910, + "stress": 6911, + "exercise": 6912, + "displayed": 6913, + "##gen": 6914, + "struggled": 6915, + "##tti": 6916, + "abroad": 6917, + "dramatic": 6918, + "wonderful": 6919, + "thereafter": 6920, + "madrid": 6921, + "component": 6922, + "widespread": 6923, + "##sed": 6924, + "tale": 6925, + "citizen": 6926, + "todd": 6927, + "monday": 6928, + "1886": 6929, + "vancouver": 6930, + "overseas": 6931, + "forcing": 6932, + "crying": 6933, + "descent": 6934, + "##ris": 6935, + "discussed": 6936, + "substantial": 6937, + "ranks": 6938, + "regime": 6939, + "1870": 6940, + "provinces": 6941, + "switch": 6942, + "drum": 6943, + "zane": 6944, + "ted": 6945, + "tribes": 6946, + "proof": 6947, + "lp": 6948, + "cream": 6949, + "researchers": 6950, + "volunteer": 6951, + "manor": 6952, + "silk": 6953, + "milan": 6954, + "donated": 6955, + "allies": 6956, + "venture": 6957, + "principle": 6958, + "delivery": 6959, + "enterprise": 6960, + "##ves": 6961, + "##ans": 6962, + "bars": 6963, + "traditionally": 6964, + "witch": 6965, + "reminded": 6966, + "copper": 6967, + "##uk": 6968, + "pete": 6969, + "inter": 6970, + "links": 6971, + "colin": 6972, + "grinned": 6973, + "elsewhere": 6974, + "competitive": 6975, + "frequent": 6976, + "##oy": 6977, + "scream": 6978, + "##hu": 6979, + "tension": 6980, + "texts": 6981, + "submarine": 6982, + "finnish": 6983, + "defending": 6984, + "defend": 6985, + "pat": 6986, + "detail": 6987, + "1884": 6988, + "affiliated": 6989, + "stuart": 6990, + "themes": 6991, + "villa": 6992, + "periods": 6993, + "tool": 6994, + "belgian": 6995, + "ruling": 6996, + "crimes": 6997, + "answers": 6998, + "folded": 6999, + "licensed": 7000, + "resort": 7001, + "demolished": 7002, + "hans": 7003, + "lucy": 7004, + "1881": 7005, + "lion": 7006, + "traded": 7007, + "photographs": 7008, + "writes": 7009, + "craig": 7010, + "##fa": 7011, + "trials": 7012, + "generated": 7013, + "beth": 7014, + "noble": 7015, + "debt": 7016, + "percentage": 7017, + "yorkshire": 7018, + "erected": 7019, + "ss": 7020, + "viewed": 7021, + "grades": 7022, + "confidence": 7023, + "ceased": 7024, + "islam": 7025, + "telephone": 7026, + "retail": 7027, + "##ible": 7028, + "chile": 7029, + "m²": 7030, + "roberts": 7031, + "sixteen": 7032, + "##ich": 7033, + "commented": 7034, + "hampshire": 7035, + "innocent": 7036, + "dual": 7037, + "pounds": 7038, + "checked": 7039, + "regulations": 7040, + "afghanistan": 7041, + "sung": 7042, + "rico": 7043, + "liberty": 7044, + "assets": 7045, + "bigger": 7046, + "options": 7047, + "angels": 7048, + "relegated": 7049, + "tribute": 7050, + "wells": 7051, + "attending": 7052, + "leaf": 7053, + "##yan": 7054, + "butler": 7055, + "romanian": 7056, + "forum": 7057, + "monthly": 7058, + "lisa": 7059, + "patterns": 7060, + "gmina": 7061, + "##tory": 7062, + "madison": 7063, + "hurricane": 7064, + "rev": 7065, + "##ians": 7066, + "bristol": 7067, + "##ula": 7068, + "elite": 7069, + "valuable": 7070, + "disaster": 7071, + "democracy": 7072, + "awareness": 7073, + "germans": 7074, + "freyja": 7075, + "##ins": 7076, + "loop": 7077, + "absolutely": 7078, + "paying": 7079, + "populations": 7080, + "maine": 7081, + "sole": 7082, + "prayer": 7083, + "spencer": 7084, + "releases": 7085, + "doorway": 7086, + "bull": 7087, + "##ani": 7088, + "lover": 7089, + "midnight": 7090, + "conclusion": 7091, + "##sson": 7092, + "thirteen": 7093, + "lily": 7094, + "mediterranean": 7095, + "##lt": 7096, + "nhl": 7097, + "proud": 7098, + "sample": 7099, + "##hill": 7100, + "drummer": 7101, + "guinea": 7102, + "##ova": 7103, + "murphy": 7104, + "climb": 7105, + "##ston": 7106, + "instant": 7107, + "attributed": 7108, + "horn": 7109, + "ain": 7110, + "railways": 7111, + "steven": 7112, + "##ao": 7113, + "autumn": 7114, + "ferry": 7115, + "opponent": 7116, + "root": 7117, + "traveling": 7118, + "secured": 7119, + "corridor": 7120, + "stretched": 7121, + "tales": 7122, + "sheet": 7123, + "trinity": 7124, + "cattle": 7125, + "helps": 7126, + "indicates": 7127, + "manhattan": 7128, + "murdered": 7129, + "fitted": 7130, + "1882": 7131, + "gentle": 7132, + "grandmother": 7133, + "mines": 7134, + "shocked": 7135, + "vegas": 7136, + "produces": 7137, + "##light": 7138, + "caribbean": 7139, + "##ou": 7140, + "belong": 7141, + "continuous": 7142, + "desperate": 7143, + "drunk": 7144, + "historically": 7145, + "trio": 7146, + "waved": 7147, + "raf": 7148, + "dealing": 7149, + "nathan": 7150, + "bat": 7151, + "murmured": 7152, + "interrupted": 7153, + "residing": 7154, + "scientist": 7155, + "pioneer": 7156, + "harold": 7157, + "aaron": 7158, + "##net": 7159, + "delta": 7160, + "attempting": 7161, + "minority": 7162, + "mini": 7163, + "believes": 7164, + "chorus": 7165, + "tend": 7166, + "lots": 7167, + "eyed": 7168, + "indoor": 7169, + "load": 7170, + "shots": 7171, + "updated": 7172, + "jail": 7173, + "##llo": 7174, + "concerning": 7175, + "connecting": 7176, + "wealth": 7177, + "##ved": 7178, + "slaves": 7179, + "arrive": 7180, + "rangers": 7181, + "sufficient": 7182, + "rebuilt": 7183, + "##wick": 7184, + "cardinal": 7185, + "flood": 7186, + "muhammad": 7187, + "whenever": 7188, + "relation": 7189, + "runners": 7190, + "moral": 7191, + "repair": 7192, + "viewers": 7193, + "arriving": 7194, + "revenge": 7195, + "punk": 7196, + "assisted": 7197, + "bath": 7198, + "fairly": 7199, + "breathe": 7200, + "lists": 7201, + "innings": 7202, + "illustrated": 7203, + "whisper": 7204, + "nearest": 7205, + "voters": 7206, + "clinton": 7207, + "ties": 7208, + "ultimate": 7209, + "screamed": 7210, + "beijing": 7211, + "lions": 7212, + "andre": 7213, + "fictional": 7214, + "gathering": 7215, + "comfort": 7216, + "radar": 7217, + "suitable": 7218, + "dismissed": 7219, + "hms": 7220, + "ban": 7221, + "pine": 7222, + "wrist": 7223, + "atmosphere": 7224, + "voivodeship": 7225, + "bid": 7226, + "timber": 7227, + "##ned": 7228, + "##nan": 7229, + "giants": 7230, + "##ane": 7231, + "cameron": 7232, + "recovery": 7233, + "uss": 7234, + "identical": 7235, + "categories": 7236, + "switched": 7237, + "serbia": 7238, + "laughter": 7239, + "noah": 7240, + "ensemble": 7241, + "therapy": 7242, + "peoples": 7243, + "touching": 7244, + "##off": 7245, + "locally": 7246, + "pearl": 7247, + "platforms": 7248, + "everywhere": 7249, + "ballet": 7250, + "tables": 7251, + "lanka": 7252, + "herbert": 7253, + "outdoor": 7254, + "toured": 7255, + "derek": 7256, + "1883": 7257, + "spaces": 7258, + "contested": 7259, + "swept": 7260, + "1878": 7261, + "exclusive": 7262, + "slight": 7263, + "connections": 7264, + "##dra": 7265, + "winds": 7266, + "prisoner": 7267, + "collective": 7268, + "bangladesh": 7269, + "tube": 7270, + "publicly": 7271, + "wealthy": 7272, + "thai": 7273, + "##ys": 7274, + "isolated": 7275, + "select": 7276, + "##ric": 7277, + "insisted": 7278, + "pen": 7279, + "fortune": 7280, + "ticket": 7281, + "spotted": 7282, + "reportedly": 7283, + "animation": 7284, + "enforcement": 7285, + "tanks": 7286, + "110": 7287, + "decides": 7288, + "wider": 7289, + "lowest": 7290, + "owen": 7291, + "##time": 7292, + "nod": 7293, + "hitting": 7294, + "##hn": 7295, + "gregory": 7296, + "furthermore": 7297, + "magazines": 7298, + "fighters": 7299, + "solutions": 7300, + "##ery": 7301, + "pointing": 7302, + "requested": 7303, + "peru": 7304, + "reed": 7305, + "chancellor": 7306, + "knights": 7307, + "mask": 7308, + "worker": 7309, + "eldest": 7310, + "flames": 7311, + "reduction": 7312, + "1860": 7313, + "volunteers": 7314, + "##tis": 7315, + "reporting": 7316, + "##hl": 7317, + "wire": 7318, + "advisory": 7319, + "endemic": 7320, + "origins": 7321, + "settlers": 7322, + "pursue": 7323, + "knock": 7324, + "consumer": 7325, + "1876": 7326, + "eu": 7327, + "compound": 7328, + "creatures": 7329, + "mansion": 7330, + "sentenced": 7331, + "ivan": 7332, + "deployed": 7333, + "guitars": 7334, + "frowned": 7335, + "involves": 7336, + "mechanism": 7337, + "kilometers": 7338, + "perspective": 7339, + "shops": 7340, + "maps": 7341, + "terminus": 7342, + "duncan": 7343, + "alien": 7344, + "fist": 7345, + "bridges": 7346, + "##pers": 7347, + "heroes": 7348, + "fed": 7349, + "derby": 7350, + "swallowed": 7351, + "##ros": 7352, + "patent": 7353, + "sara": 7354, + "illness": 7355, + "characterized": 7356, + "adventures": 7357, + "slide": 7358, + "hawaii": 7359, + "jurisdiction": 7360, + "##op": 7361, + "organised": 7362, + "##side": 7363, + "adelaide": 7364, + "walks": 7365, + "biology": 7366, + "se": 7367, + "##ties": 7368, + "rogers": 7369, + "swing": 7370, + "tightly": 7371, + "boundaries": 7372, + "##rie": 7373, + "prepare": 7374, + "implementation": 7375, + "stolen": 7376, + "##sha": 7377, + "certified": 7378, + "colombia": 7379, + "edwards": 7380, + "garage": 7381, + "##mm": 7382, + "recalled": 7383, + "##ball": 7384, + "rage": 7385, + "harm": 7386, + "nigeria": 7387, + "breast": 7388, + "##ren": 7389, + "furniture": 7390, + "pupils": 7391, + "settle": 7392, + "##lus": 7393, + "cuba": 7394, + "balls": 7395, + "client": 7396, + "alaska": 7397, + "21st": 7398, + "linear": 7399, + "thrust": 7400, + "celebration": 7401, + "latino": 7402, + "genetic": 7403, + "terror": 7404, + "##cia": 7405, + "##ening": 7406, + "lightning": 7407, + "fee": 7408, + "witness": 7409, + "lodge": 7410, + "establishing": 7411, + "skull": 7412, + "##ique": 7413, + "earning": 7414, + "hood": 7415, + "##ei": 7416, + "rebellion": 7417, + "wang": 7418, + "sporting": 7419, + "warned": 7420, + "missile": 7421, + "devoted": 7422, + "activist": 7423, + "porch": 7424, + "worship": 7425, + "fourteen": 7426, + "package": 7427, + "1871": 7428, + "decorated": 7429, + "##shire": 7430, + "housed": 7431, + "##ock": 7432, + "chess": 7433, + "sailed": 7434, + "doctors": 7435, + "oscar": 7436, + "joan": 7437, + "treat": 7438, + "garcia": 7439, + "harbour": 7440, + "jeremy": 7441, + "##ire": 7442, + "traditions": 7443, + "dominant": 7444, + "jacques": 7445, + "##gon": 7446, + "##wan": 7447, + "relocated": 7448, + "1879": 7449, + "amendment": 7450, + "sized": 7451, + "companion": 7452, + "simultaneously": 7453, + "volleyball": 7454, + "spun": 7455, + "acre": 7456, + "increases": 7457, + "stopping": 7458, + "loves": 7459, + "belongs": 7460, + "affect": 7461, + "drafted": 7462, + "tossed": 7463, + "scout": 7464, + "battles": 7465, + "1875": 7466, + "filming": 7467, + "shoved": 7468, + "munich": 7469, + "tenure": 7470, + "vertical": 7471, + "romance": 7472, + "pc": 7473, + "##cher": 7474, + "argue": 7475, + "##ical": 7476, + "craft": 7477, + "ranging": 7478, + "www": 7479, + "opens": 7480, + "honest": 7481, + "tyler": 7482, + "yesterday": 7483, + "virtual": 7484, + "##let": 7485, + "muslims": 7486, + "reveal": 7487, + "snake": 7488, + "immigrants": 7489, + "radical": 7490, + "screaming": 7491, + "speakers": 7492, + "firing": 7493, + "saving": 7494, + "belonging": 7495, + "ease": 7496, + "lighting": 7497, + "prefecture": 7498, + "blame": 7499, + "farmer": 7500, + "hungry": 7501, + "grows": 7502, + "rubbed": 7503, + "beam": 7504, + "sur": 7505, + "subsidiary": 7506, + "##cha": 7507, + "armenian": 7508, + "sao": 7509, + "dropping": 7510, + "conventional": 7511, + "##fer": 7512, + "microsoft": 7513, + "reply": 7514, + "qualify": 7515, + "spots": 7516, + "1867": 7517, + "sweat": 7518, + "festivals": 7519, + "##ken": 7520, + "immigration": 7521, + "physician": 7522, + "discover": 7523, + "exposure": 7524, + "sandy": 7525, + "explanation": 7526, + "isaac": 7527, + "implemented": 7528, + "##fish": 7529, + "hart": 7530, + "initiated": 7531, + "connect": 7532, + "stakes": 7533, + "presents": 7534, + "heights": 7535, + "householder": 7536, + "pleased": 7537, + "tourist": 7538, + "regardless": 7539, + "slip": 7540, + "closest": 7541, + "##ction": 7542, + "surely": 7543, + "sultan": 7544, + "brings": 7545, + "riley": 7546, + "preparation": 7547, + "aboard": 7548, + "slammed": 7549, + "baptist": 7550, + "experiment": 7551, + "ongoing": 7552, + "interstate": 7553, + "organic": 7554, + "playoffs": 7555, + "##ika": 7556, + "1877": 7557, + "130": 7558, + "##tar": 7559, + "hindu": 7560, + "error": 7561, + "tours": 7562, + "tier": 7563, + "plenty": 7564, + "arrangements": 7565, + "talks": 7566, + "trapped": 7567, + "excited": 7568, + "sank": 7569, + "ho": 7570, + "athens": 7571, + "1872": 7572, + "denver": 7573, + "welfare": 7574, + "suburb": 7575, + "athletes": 7576, + "trick": 7577, + "diverse": 7578, + "belly": 7579, + "exclusively": 7580, + "yelled": 7581, + "1868": 7582, + "##med": 7583, + "conversion": 7584, + "##ette": 7585, + "1874": 7586, + "internationally": 7587, + "computers": 7588, + "conductor": 7589, + "abilities": 7590, + "sensitive": 7591, + "hello": 7592, + "dispute": 7593, + "measured": 7594, + "globe": 7595, + "rocket": 7596, + "prices": 7597, + "amsterdam": 7598, + "flights": 7599, + "tigers": 7600, + "inn": 7601, + "municipalities": 7602, + "emotion": 7603, + "references": 7604, + "3d": 7605, + "##mus": 7606, + "explains": 7607, + "airlines": 7608, + "manufactured": 7609, + "pm": 7610, + "archaeological": 7611, + "1873": 7612, + "interpretation": 7613, + "devon": 7614, + "comment": 7615, + "##ites": 7616, + "settlements": 7617, + "kissing": 7618, + "absolute": 7619, + "improvement": 7620, + "suite": 7621, + "impressed": 7622, + "barcelona": 7623, + "sullivan": 7624, + "jefferson": 7625, + "towers": 7626, + "jesse": 7627, + "julie": 7628, + "##tin": 7629, + "##lu": 7630, + "grandson": 7631, + "hi": 7632, + "gauge": 7633, + "regard": 7634, + "rings": 7635, + "interviews": 7636, + "trace": 7637, + "raymond": 7638, + "thumb": 7639, + "departments": 7640, + "burns": 7641, + "serial": 7642, + "bulgarian": 7643, + "scores": 7644, + "demonstrated": 7645, + "##ix": 7646, + "1866": 7647, + "kyle": 7648, + "alberta": 7649, + "underneath": 7650, + "romanized": 7651, + "##ward": 7652, + "relieved": 7653, + "acquisition": 7654, + "phrase": 7655, + "cliff": 7656, + "reveals": 7657, + "han": 7658, + "cuts": 7659, + "merger": 7660, + "custom": 7661, + "##dar": 7662, + "nee": 7663, + "gilbert": 7664, + "graduation": 7665, + "##nts": 7666, + "assessment": 7667, + "cafe": 7668, + "difficulty": 7669, + "demands": 7670, + "swung": 7671, + "democrat": 7672, + "jennifer": 7673, + "commons": 7674, + "1940s": 7675, + "grove": 7676, + "##yo": 7677, + "completing": 7678, + "focuses": 7679, + "sum": 7680, + "substitute": 7681, + "bearing": 7682, + "stretch": 7683, + "reception": 7684, + "##py": 7685, + "reflected": 7686, + "essentially": 7687, + "destination": 7688, + "pairs": 7689, + "##ched": 7690, + "survival": 7691, + "resource": 7692, + "##bach": 7693, + "promoting": 7694, + "doubles": 7695, + "messages": 7696, + "tear": 7697, + "##down": 7698, + "##fully": 7699, + "parade": 7700, + "florence": 7701, + "harvey": 7702, + "incumbent": 7703, + "partial": 7704, + "framework": 7705, + "900": 7706, + "pedro": 7707, + "frozen": 7708, + "procedure": 7709, + "olivia": 7710, + "controls": 7711, + "##mic": 7712, + "shelter": 7713, + "personally": 7714, + "temperatures": 7715, + "##od": 7716, + "brisbane": 7717, + "tested": 7718, + "sits": 7719, + "marble": 7720, + "comprehensive": 7721, + "oxygen": 7722, + "leonard": 7723, + "##kov": 7724, + "inaugural": 7725, + "iranian": 7726, + "referring": 7727, + "quarters": 7728, + "attitude": 7729, + "##ivity": 7730, + "mainstream": 7731, + "lined": 7732, + "mars": 7733, + "dakota": 7734, + "norfolk": 7735, + "unsuccessful": 7736, + "##°": 7737, + "explosion": 7738, + "helicopter": 7739, + "congressional": 7740, + "##sing": 7741, + "inspector": 7742, + "bitch": 7743, + "seal": 7744, + "departed": 7745, + "divine": 7746, + "##ters": 7747, + "coaching": 7748, + "examination": 7749, + "punishment": 7750, + "manufacturer": 7751, + "sink": 7752, + "columns": 7753, + "unincorporated": 7754, + "signals": 7755, + "nevada": 7756, + "squeezed": 7757, + "dylan": 7758, + "dining": 7759, + "photos": 7760, + "martial": 7761, + "manuel": 7762, + "eighteen": 7763, + "elevator": 7764, + "brushed": 7765, + "plates": 7766, + "ministers": 7767, + "ivy": 7768, + "congregation": 7769, + "##len": 7770, + "slept": 7771, + "specialized": 7772, + "taxes": 7773, + "curve": 7774, + "restricted": 7775, + "negotiations": 7776, + "likes": 7777, + "statistical": 7778, + "arnold": 7779, + "inspiration": 7780, + "execution": 7781, + "bold": 7782, + "intermediate": 7783, + "significance": 7784, + "margin": 7785, + "ruler": 7786, + "wheels": 7787, + "gothic": 7788, + "intellectual": 7789, + "dependent": 7790, + "listened": 7791, + "eligible": 7792, + "buses": 7793, + "widow": 7794, + "syria": 7795, + "earn": 7796, + "cincinnati": 7797, + "collapsed": 7798, + "recipient": 7799, + "secrets": 7800, + "accessible": 7801, + "philippine": 7802, + "maritime": 7803, + "goddess": 7804, + "clerk": 7805, + "surrender": 7806, + "breaks": 7807, + "playoff": 7808, + "database": 7809, + "##ified": 7810, + "##lon": 7811, + "ideal": 7812, + "beetle": 7813, + "aspect": 7814, + "soap": 7815, + "regulation": 7816, + "strings": 7817, + "expand": 7818, + "anglo": 7819, + "shorter": 7820, + "crosses": 7821, + "retreat": 7822, + "tough": 7823, + "coins": 7824, + "wallace": 7825, + "directions": 7826, + "pressing": 7827, + "##oon": 7828, + "shipping": 7829, + "locomotives": 7830, + "comparison": 7831, + "topics": 7832, + "nephew": 7833, + "##mes": 7834, + "distinction": 7835, + "honors": 7836, + "travelled": 7837, + "sierra": 7838, + "ibn": 7839, + "##over": 7840, + "fortress": 7841, + "sa": 7842, + "recognised": 7843, + "carved": 7844, + "1869": 7845, + "clients": 7846, + "##dan": 7847, + "intent": 7848, + "##mar": 7849, + "coaches": 7850, + "describing": 7851, + "bread": 7852, + "##ington": 7853, + "beaten": 7854, + "northwestern": 7855, + "##ona": 7856, + "merit": 7857, + "youtube": 7858, + "collapse": 7859, + "challenges": 7860, + "em": 7861, + "historians": 7862, + "objective": 7863, + "submitted": 7864, + "virus": 7865, + "attacking": 7866, + "drake": 7867, + "assume": 7868, + "##ere": 7869, + "diseases": 7870, + "marc": 7871, + "stem": 7872, + "leeds": 7873, + "##cus": 7874, + "##ab": 7875, + "farming": 7876, + "glasses": 7877, + "##lock": 7878, + "visits": 7879, + "nowhere": 7880, + "fellowship": 7881, + "relevant": 7882, + "carries": 7883, + "restaurants": 7884, + "experiments": 7885, + "101": 7886, + "constantly": 7887, + "bases": 7888, + "targets": 7889, + "shah": 7890, + "tenth": 7891, + "opponents": 7892, + "verse": 7893, + "territorial": 7894, + "##ira": 7895, + "writings": 7896, + "corruption": 7897, + "##hs": 7898, + "instruction": 7899, + "inherited": 7900, + "reverse": 7901, + "emphasis": 7902, + "##vic": 7903, + "employee": 7904, + "arch": 7905, + "keeps": 7906, + "rabbi": 7907, + "watson": 7908, + "payment": 7909, + "uh": 7910, + "##ala": 7911, + "nancy": 7912, + "##tre": 7913, + "venice": 7914, + "fastest": 7915, + "sexy": 7916, + "banned": 7917, + "adrian": 7918, + "properly": 7919, + "ruth": 7920, + "touchdown": 7921, + "dollar": 7922, + "boards": 7923, + "metre": 7924, + "circles": 7925, + "edges": 7926, + "favour": 7927, + "comments": 7928, + "ok": 7929, + "travels": 7930, + "liberation": 7931, + "scattered": 7932, + "firmly": 7933, + "##ular": 7934, + "holland": 7935, + "permitted": 7936, + "diesel": 7937, + "kenya": 7938, + "den": 7939, + "originated": 7940, + "##ral": 7941, + "demons": 7942, + "resumed": 7943, + "dragged": 7944, + "rider": 7945, + "##rus": 7946, + "servant": 7947, + "blinked": 7948, + "extend": 7949, + "torn": 7950, + "##ias": 7951, + "##sey": 7952, + "input": 7953, + "meal": 7954, + "everybody": 7955, + "cylinder": 7956, + "kinds": 7957, + "camps": 7958, + "##fe": 7959, + "bullet": 7960, + "logic": 7961, + "##wn": 7962, + "croatian": 7963, + "evolved": 7964, + "healthy": 7965, + "fool": 7966, + "chocolate": 7967, + "wise": 7968, + "preserve": 7969, + "pradesh": 7970, + "##ess": 7971, + "respective": 7972, + "1850": 7973, + "##ew": 7974, + "chicken": 7975, + "artificial": 7976, + "gross": 7977, + "corresponding": 7978, + "convicted": 7979, + "cage": 7980, + "caroline": 7981, + "dialogue": 7982, + "##dor": 7983, + "narrative": 7984, + "stranger": 7985, + "mario": 7986, + "br": 7987, + "christianity": 7988, + "failing": 7989, + "trent": 7990, + "commanding": 7991, + "buddhist": 7992, + "1848": 7993, + "maurice": 7994, + "focusing": 7995, + "yale": 7996, + "bike": 7997, + "altitude": 7998, + "##ering": 7999, + "mouse": 8000, + "revised": 8001, + "##sley": 8002, + "veteran": 8003, + "##ig": 8004, + "pulls": 8005, + "theology": 8006, + "crashed": 8007, + "campaigns": 8008, + "legion": 8009, + "##ability": 8010, + "drag": 8011, + "excellence": 8012, + "customer": 8013, + "cancelled": 8014, + "intensity": 8015, + "excuse": 8016, + "##lar": 8017, + "liga": 8018, + "participating": 8019, + "contributing": 8020, + "printing": 8021, + "##burn": 8022, + "variable": 8023, + "##rk": 8024, + "curious": 8025, + "bin": 8026, + "legacy": 8027, + "renaissance": 8028, + "##my": 8029, + "symptoms": 8030, + "binding": 8031, + "vocalist": 8032, + "dancer": 8033, + "##nie": 8034, + "grammar": 8035, + "gospel": 8036, + "democrats": 8037, + "ya": 8038, + "enters": 8039, + "sc": 8040, + "diplomatic": 8041, + "hitler": 8042, + "##ser": 8043, + "clouds": 8044, + "mathematical": 8045, + "quit": 8046, + "defended": 8047, + "oriented": 8048, + "##heim": 8049, + "fundamental": 8050, + "hardware": 8051, + "impressive": 8052, + "equally": 8053, + "convince": 8054, + "confederate": 8055, + "guilt": 8056, + "chuck": 8057, + "sliding": 8058, + "##ware": 8059, + "magnetic": 8060, + "narrowed": 8061, + "petersburg": 8062, + "bulgaria": 8063, + "otto": 8064, + "phd": 8065, + "skill": 8066, + "##ama": 8067, + "reader": 8068, + "hopes": 8069, + "pitcher": 8070, + "reservoir": 8071, + "hearts": 8072, + "automatically": 8073, + "expecting": 8074, + "mysterious": 8075, + "bennett": 8076, + "extensively": 8077, + "imagined": 8078, + "seeds": 8079, + "monitor": 8080, + "fix": 8081, + "##ative": 8082, + "journalism": 8083, + "struggling": 8084, + "signature": 8085, + "ranch": 8086, + "encounter": 8087, + "photographer": 8088, + "observation": 8089, + "protests": 8090, + "##pin": 8091, + "influences": 8092, + "##hr": 8093, + "calendar": 8094, + "##all": 8095, + "cruz": 8096, + "croatia": 8097, + "locomotive": 8098, + "hughes": 8099, + "naturally": 8100, + "shakespeare": 8101, + "basement": 8102, + "hook": 8103, + "uncredited": 8104, + "faded": 8105, + "theories": 8106, + "approaches": 8107, + "dare": 8108, + "phillips": 8109, + "filling": 8110, + "fury": 8111, + "obama": 8112, + "##ain": 8113, + "efficient": 8114, + "arc": 8115, + "deliver": 8116, + "min": 8117, + "raid": 8118, + "breeding": 8119, + "inducted": 8120, + "leagues": 8121, + "efficiency": 8122, + "axis": 8123, + "montana": 8124, + "eagles": 8125, + "##ked": 8126, + "supplied": 8127, + "instructions": 8128, + "karen": 8129, + "picking": 8130, + "indicating": 8131, + "trap": 8132, + "anchor": 8133, + "practically": 8134, + "christians": 8135, + "tomb": 8136, + "vary": 8137, + "occasional": 8138, + "electronics": 8139, + "lords": 8140, + "readers": 8141, + "newcastle": 8142, + "faint": 8143, + "innovation": 8144, + "collect": 8145, + "situations": 8146, + "engagement": 8147, + "160": 8148, + "claude": 8149, + "mixture": 8150, + "##feld": 8151, + "peer": 8152, + "tissue": 8153, + "logo": 8154, + "lean": 8155, + "##ration": 8156, + "°f": 8157, + "floors": 8158, + "##ven": 8159, + "architects": 8160, + "reducing": 8161, + "##our": 8162, + "##ments": 8163, + "rope": 8164, + "1859": 8165, + "ottawa": 8166, + "##har": 8167, + "samples": 8168, + "banking": 8169, + "declaration": 8170, + "proteins": 8171, + "resignation": 8172, + "francois": 8173, + "saudi": 8174, + "advocate": 8175, + "exhibited": 8176, + "armor": 8177, + "twins": 8178, + "divorce": 8179, + "##ras": 8180, + "abraham": 8181, + "reviewed": 8182, + "jo": 8183, + "temporarily": 8184, + "matrix": 8185, + "physically": 8186, + "pulse": 8187, + "curled": 8188, + "##ena": 8189, + "difficulties": 8190, + "bengal": 8191, + "usage": 8192, + "##ban": 8193, + "annie": 8194, + "riders": 8195, + "certificate": 8196, + "##pi": 8197, + "holes": 8198, + "warsaw": 8199, + "distinctive": 8200, + "jessica": 8201, + "##mon": 8202, + "mutual": 8203, + "1857": 8204, + "customs": 8205, + "circular": 8206, + "eugene": 8207, + "removal": 8208, + "loaded": 8209, + "mere": 8210, + "vulnerable": 8211, + "depicted": 8212, + "generations": 8213, + "dame": 8214, + "heir": 8215, + "enormous": 8216, + "lightly": 8217, + "climbing": 8218, + "pitched": 8219, + "lessons": 8220, + "pilots": 8221, + "nepal": 8222, + "ram": 8223, + "google": 8224, + "preparing": 8225, + "brad": 8226, + "louise": 8227, + "renowned": 8228, + "##₂": 8229, + "liam": 8230, + "##ably": 8231, + "plaza": 8232, + "shaw": 8233, + "sophie": 8234, + "brilliant": 8235, + "bills": 8236, + "##bar": 8237, + "##nik": 8238, + "fucking": 8239, + "mainland": 8240, + "server": 8241, + "pleasant": 8242, + "seized": 8243, + "veterans": 8244, + "jerked": 8245, + "fail": 8246, + "beta": 8247, + "brush": 8248, + "radiation": 8249, + "stored": 8250, + "warmth": 8251, + "southeastern": 8252, + "nate": 8253, + "sin": 8254, + "raced": 8255, + "berkeley": 8256, + "joke": 8257, + "athlete": 8258, + "designation": 8259, + "trunk": 8260, + "##low": 8261, + "roland": 8262, + "qualification": 8263, + "archives": 8264, + "heels": 8265, + "artwork": 8266, + "receives": 8267, + "judicial": 8268, + "reserves": 8269, + "##bed": 8270, + "woke": 8271, + "installation": 8272, + "abu": 8273, + "floating": 8274, + "fake": 8275, + "lesser": 8276, + "excitement": 8277, + "interface": 8278, + "concentrated": 8279, + "addressed": 8280, + "characteristic": 8281, + "amanda": 8282, + "saxophone": 8283, + "monk": 8284, + "auto": 8285, + "##bus": 8286, + "releasing": 8287, + "egg": 8288, + "dies": 8289, + "interaction": 8290, + "defender": 8291, + "ce": 8292, + "outbreak": 8293, + "glory": 8294, + "loving": 8295, + "##bert": 8296, + "sequel": 8297, + "consciousness": 8298, + "http": 8299, + "awake": 8300, + "ski": 8301, + "enrolled": 8302, + "##ress": 8303, + "handling": 8304, + "rookie": 8305, + "brow": 8306, + "somebody": 8307, + "biography": 8308, + "warfare": 8309, + "amounts": 8310, + "contracts": 8311, + "presentation": 8312, + "fabric": 8313, + "dissolved": 8314, + "challenged": 8315, + "meter": 8316, + "psychological": 8317, + "lt": 8318, + "elevated": 8319, + "rally": 8320, + "accurate": 8321, + "##tha": 8322, + "hospitals": 8323, + "undergraduate": 8324, + "specialist": 8325, + "venezuela": 8326, + "exhibit": 8327, + "shed": 8328, + "nursing": 8329, + "protestant": 8330, + "fluid": 8331, + "structural": 8332, + "footage": 8333, + "jared": 8334, + "consistent": 8335, + "prey": 8336, + "##ska": 8337, + "succession": 8338, + "reflect": 8339, + "exile": 8340, + "lebanon": 8341, + "wiped": 8342, + "suspect": 8343, + "shanghai": 8344, + "resting": 8345, + "integration": 8346, + "preservation": 8347, + "marvel": 8348, + "variant": 8349, + "pirates": 8350, + "sheep": 8351, + "rounded": 8352, + "capita": 8353, + "sailing": 8354, + "colonies": 8355, + "manuscript": 8356, + "deemed": 8357, + "variations": 8358, + "clarke": 8359, + "functional": 8360, + "emerging": 8361, + "boxing": 8362, + "relaxed": 8363, + "curse": 8364, + "azerbaijan": 8365, + "heavyweight": 8366, + "nickname": 8367, + "editorial": 8368, + "rang": 8369, + "grid": 8370, + "tightened": 8371, + "earthquake": 8372, + "flashed": 8373, + "miguel": 8374, + "rushing": 8375, + "##ches": 8376, + "improvements": 8377, + "boxes": 8378, + "brooks": 8379, + "180": 8380, + "consumption": 8381, + "molecular": 8382, + "felix": 8383, + "societies": 8384, + "repeatedly": 8385, + "variation": 8386, + "aids": 8387, + "civic": 8388, + "graphics": 8389, + "professionals": 8390, + "realm": 8391, + "autonomous": 8392, + "receiver": 8393, + "delayed": 8394, + "workshop": 8395, + "militia": 8396, + "chairs": 8397, + "trump": 8398, + "canyon": 8399, + "##point": 8400, + "harsh": 8401, + "extending": 8402, + "lovely": 8403, + "happiness": 8404, + "##jan": 8405, + "stake": 8406, + "eyebrows": 8407, + "embassy": 8408, + "wellington": 8409, + "hannah": 8410, + "##ella": 8411, + "sony": 8412, + "corners": 8413, + "bishops": 8414, + "swear": 8415, + "cloth": 8416, + "contents": 8417, + "xi": 8418, + "namely": 8419, + "commenced": 8420, + "1854": 8421, + "stanford": 8422, + "nashville": 8423, + "courage": 8424, + "graphic": 8425, + "commitment": 8426, + "garrison": 8427, + "##bin": 8428, + "hamlet": 8429, + "clearing": 8430, + "rebels": 8431, + "attraction": 8432, + "literacy": 8433, + "cooking": 8434, + "ruins": 8435, + "temples": 8436, + "jenny": 8437, + "humanity": 8438, + "celebrate": 8439, + "hasn": 8440, + "freight": 8441, + "sixty": 8442, + "rebel": 8443, + "bastard": 8444, + "##art": 8445, + "newton": 8446, + "##ada": 8447, + "deer": 8448, + "##ges": 8449, + "##ching": 8450, + "smiles": 8451, + "delaware": 8452, + "singers": 8453, + "##ets": 8454, + "approaching": 8455, + "assists": 8456, + "flame": 8457, + "##ph": 8458, + "boulevard": 8459, + "barrel": 8460, + "planted": 8461, + "##ome": 8462, + "pursuit": 8463, + "##sia": 8464, + "consequences": 8465, + "posts": 8466, + "shallow": 8467, + "invitation": 8468, + "rode": 8469, + "depot": 8470, + "ernest": 8471, + "kane": 8472, + "rod": 8473, + "concepts": 8474, + "preston": 8475, + "topic": 8476, + "chambers": 8477, + "striking": 8478, + "blast": 8479, + "arrives": 8480, + "descendants": 8481, + "montgomery": 8482, + "ranges": 8483, + "worlds": 8484, + "##lay": 8485, + "##ari": 8486, + "span": 8487, + "chaos": 8488, + "praise": 8489, + "##ag": 8490, + "fewer": 8491, + "1855": 8492, + "sanctuary": 8493, + "mud": 8494, + "fbi": 8495, + "##ions": 8496, + "programmes": 8497, + "maintaining": 8498, + "unity": 8499, + "harper": 8500, + "bore": 8501, + "handsome": 8502, + "closure": 8503, + "tournaments": 8504, + "thunder": 8505, + "nebraska": 8506, + "linda": 8507, + "facade": 8508, + "puts": 8509, + "satisfied": 8510, + "argentine": 8511, + "dale": 8512, + "cork": 8513, + "dome": 8514, + "panama": 8515, + "##yl": 8516, + "1858": 8517, + "tasks": 8518, + "experts": 8519, + "##ates": 8520, + "feeding": 8521, + "equation": 8522, + "##las": 8523, + "##ida": 8524, + "##tu": 8525, + "engage": 8526, + "bryan": 8527, + "##ax": 8528, + "um": 8529, + "quartet": 8530, + "melody": 8531, + "disbanded": 8532, + "sheffield": 8533, + "blocked": 8534, + "gasped": 8535, + "delay": 8536, + "kisses": 8537, + "maggie": 8538, + "connects": 8539, + "##non": 8540, + "sts": 8541, + "poured": 8542, + "creator": 8543, + "publishers": 8544, + "##we": 8545, + "guided": 8546, + "ellis": 8547, + "extinct": 8548, + "hug": 8549, + "gaining": 8550, + "##ord": 8551, + "complicated": 8552, + "##bility": 8553, + "poll": 8554, + "clenched": 8555, + "investigate": 8556, + "##use": 8557, + "thereby": 8558, + "quantum": 8559, + "spine": 8560, + "cdp": 8561, + "humor": 8562, + "kills": 8563, + "administered": 8564, + "semifinals": 8565, + "##du": 8566, + "encountered": 8567, + "ignore": 8568, + "##bu": 8569, + "commentary": 8570, + "##maker": 8571, + "bother": 8572, + "roosevelt": 8573, + "140": 8574, + "plains": 8575, + "halfway": 8576, + "flowing": 8577, + "cultures": 8578, + "crack": 8579, + "imprisoned": 8580, + "neighboring": 8581, + "airline": 8582, + "##ses": 8583, + "##view": 8584, + "##mate": 8585, + "##ec": 8586, + "gather": 8587, + "wolves": 8588, + "marathon": 8589, + "transformed": 8590, + "##ill": 8591, + "cruise": 8592, + "organisations": 8593, + "carol": 8594, + "punch": 8595, + "exhibitions": 8596, + "numbered": 8597, + "alarm": 8598, + "ratings": 8599, + "daddy": 8600, + "silently": 8601, + "##stein": 8602, + "queens": 8603, + "colours": 8604, + "impression": 8605, + "guidance": 8606, + "liu": 8607, + "tactical": 8608, + "##rat": 8609, + "marshal": 8610, + "della": 8611, + "arrow": 8612, + "##ings": 8613, + "rested": 8614, + "feared": 8615, + "tender": 8616, + "owns": 8617, + "bitter": 8618, + "advisor": 8619, + "escort": 8620, + "##ides": 8621, + "spare": 8622, + "farms": 8623, + "grants": 8624, + "##ene": 8625, + "dragons": 8626, + "encourage": 8627, + "colleagues": 8628, + "cameras": 8629, + "##und": 8630, + "sucked": 8631, + "pile": 8632, + "spirits": 8633, + "prague": 8634, + "statements": 8635, + "suspension": 8636, + "landmark": 8637, + "fence": 8638, + "torture": 8639, + "recreation": 8640, + "bags": 8641, + "permanently": 8642, + "survivors": 8643, + "pond": 8644, + "spy": 8645, + "predecessor": 8646, + "bombing": 8647, + "coup": 8648, + "##og": 8649, + "protecting": 8650, + "transformation": 8651, + "glow": 8652, + "##lands": 8653, + "##book": 8654, + "dug": 8655, + "priests": 8656, + "andrea": 8657, + "feat": 8658, + "barn": 8659, + "jumping": 8660, + "##chen": 8661, + "##ologist": 8662, + "##con": 8663, + "casualties": 8664, + "stern": 8665, + "auckland": 8666, + "pipe": 8667, + "serie": 8668, + "revealing": 8669, + "ba": 8670, + "##bel": 8671, + "trevor": 8672, + "mercy": 8673, + "spectrum": 8674, + "yang": 8675, + "consist": 8676, + "governing": 8677, + "collaborated": 8678, + "possessed": 8679, + "epic": 8680, + "comprises": 8681, + "blew": 8682, + "shane": 8683, + "##ack": 8684, + "lopez": 8685, + "honored": 8686, + "magical": 8687, + "sacrifice": 8688, + "judgment": 8689, + "perceived": 8690, + "hammer": 8691, + "mtv": 8692, + "baronet": 8693, + "tune": 8694, + "das": 8695, + "missionary": 8696, + "sheets": 8697, + "350": 8698, + "neutral": 8699, + "oral": 8700, + "threatening": 8701, + "attractive": 8702, + "shade": 8703, + "aims": 8704, + "seminary": 8705, + "##master": 8706, + "estates": 8707, + "1856": 8708, + "michel": 8709, + "wounds": 8710, + "refugees": 8711, + "manufacturers": 8712, + "##nic": 8713, + "mercury": 8714, + "syndrome": 8715, + "porter": 8716, + "##iya": 8717, + "##din": 8718, + "hamburg": 8719, + "identification": 8720, + "upstairs": 8721, + "purse": 8722, + "widened": 8723, + "pause": 8724, + "cared": 8725, + "breathed": 8726, + "affiliate": 8727, + "santiago": 8728, + "prevented": 8729, + "celtic": 8730, + "fisher": 8731, + "125": 8732, + "recruited": 8733, + "byzantine": 8734, + "reconstruction": 8735, + "farther": 8736, + "##mp": 8737, + "diet": 8738, + "sake": 8739, + "au": 8740, + "spite": 8741, + "sensation": 8742, + "##ert": 8743, + "blank": 8744, + "separation": 8745, + "105": 8746, + "##hon": 8747, + "vladimir": 8748, + "armies": 8749, + "anime": 8750, + "##lie": 8751, + "accommodate": 8752, + "orbit": 8753, + "cult": 8754, + "sofia": 8755, + "archive": 8756, + "##ify": 8757, + "##box": 8758, + "founders": 8759, + "sustained": 8760, + "disorder": 8761, + "honours": 8762, + "northeastern": 8763, + "mia": 8764, + "crops": 8765, + "violet": 8766, + "threats": 8767, + "blanket": 8768, + "fires": 8769, + "canton": 8770, + "followers": 8771, + "southwestern": 8772, + "prototype": 8773, + "voyage": 8774, + "assignment": 8775, + "altered": 8776, + "moderate": 8777, + "protocol": 8778, + "pistol": 8779, + "##eo": 8780, + "questioned": 8781, + "brass": 8782, + "lifting": 8783, + "1852": 8784, + "math": 8785, + "authored": 8786, + "##ual": 8787, + "doug": 8788, + "dimensional": 8789, + "dynamic": 8790, + "##san": 8791, + "1851": 8792, + "pronounced": 8793, + "grateful": 8794, + "quest": 8795, + "uncomfortable": 8796, + "boom": 8797, + "presidency": 8798, + "stevens": 8799, + "relating": 8800, + "politicians": 8801, + "chen": 8802, + "barrier": 8803, + "quinn": 8804, + "diana": 8805, + "mosque": 8806, + "tribal": 8807, + "cheese": 8808, + "palmer": 8809, + "portions": 8810, + "sometime": 8811, + "chester": 8812, + "treasure": 8813, + "wu": 8814, + "bend": 8815, + "download": 8816, + "millions": 8817, + "reforms": 8818, + "registration": 8819, + "##osa": 8820, + "consequently": 8821, + "monitoring": 8822, + "ate": 8823, + "preliminary": 8824, + "brandon": 8825, + "invented": 8826, + "ps": 8827, + "eaten": 8828, + "exterior": 8829, + "intervention": 8830, + "ports": 8831, + "documented": 8832, + "log": 8833, + "displays": 8834, + "lecture": 8835, + "sally": 8836, + "favourite": 8837, + "##itz": 8838, + "vermont": 8839, + "lo": 8840, + "invisible": 8841, + "isle": 8842, + "breed": 8843, + "##ator": 8844, + "journalists": 8845, + "relay": 8846, + "speaks": 8847, + "backward": 8848, + "explore": 8849, + "midfielder": 8850, + "actively": 8851, + "stefan": 8852, + "procedures": 8853, + "cannon": 8854, + "blond": 8855, + "kenneth": 8856, + "centered": 8857, + "servants": 8858, + "chains": 8859, + "libraries": 8860, + "malcolm": 8861, + "essex": 8862, + "henri": 8863, + "slavery": 8864, + "##hal": 8865, + "facts": 8866, + "fairy": 8867, + "coached": 8868, + "cassie": 8869, + "cats": 8870, + "washed": 8871, + "cop": 8872, + "##fi": 8873, + "announcement": 8874, + "item": 8875, + "2000s": 8876, + "vinyl": 8877, + "activated": 8878, + "marco": 8879, + "frontier": 8880, + "growled": 8881, + "curriculum": 8882, + "##das": 8883, + "loyal": 8884, + "accomplished": 8885, + "leslie": 8886, + "ritual": 8887, + "kenny": 8888, + "##00": 8889, + "vii": 8890, + "napoleon": 8891, + "hollow": 8892, + "hybrid": 8893, + "jungle": 8894, + "stationed": 8895, + "friedrich": 8896, + "counted": 8897, + "##ulated": 8898, + "platinum": 8899, + "theatrical": 8900, + "seated": 8901, + "col": 8902, + "rubber": 8903, + "glen": 8904, + "1840": 8905, + "diversity": 8906, + "healing": 8907, + "extends": 8908, + "id": 8909, + "provisions": 8910, + "administrator": 8911, + "columbus": 8912, + "##oe": 8913, + "tributary": 8914, + "te": 8915, + "assured": 8916, + "org": 8917, + "##uous": 8918, + "prestigious": 8919, + "examined": 8920, + "lectures": 8921, + "grammy": 8922, + "ronald": 8923, + "associations": 8924, + "bailey": 8925, + "allan": 8926, + "essays": 8927, + "flute": 8928, + "believing": 8929, + "consultant": 8930, + "proceedings": 8931, + "travelling": 8932, + "1853": 8933, + "kit": 8934, + "kerala": 8935, + "yugoslavia": 8936, + "buddy": 8937, + "methodist": 8938, + "##ith": 8939, + "burial": 8940, + "centres": 8941, + "batman": 8942, + "##nda": 8943, + "discontinued": 8944, + "bo": 8945, + "dock": 8946, + "stockholm": 8947, + "lungs": 8948, + "severely": 8949, + "##nk": 8950, + "citing": 8951, + "manga": 8952, + "##ugh": 8953, + "steal": 8954, + "mumbai": 8955, + "iraqi": 8956, + "robot": 8957, + "celebrity": 8958, + "bride": 8959, + "broadcasts": 8960, + "abolished": 8961, + "pot": 8962, + "joel": 8963, + "overhead": 8964, + "franz": 8965, + "packed": 8966, + "reconnaissance": 8967, + "johann": 8968, + "acknowledged": 8969, + "introduce": 8970, + "handled": 8971, + "doctorate": 8972, + "developments": 8973, + "drinks": 8974, + "alley": 8975, + "palestine": 8976, + "##nis": 8977, + "##aki": 8978, + "proceeded": 8979, + "recover": 8980, + "bradley": 8981, + "grain": 8982, + "patch": 8983, + "afford": 8984, + "infection": 8985, + "nationalist": 8986, + "legendary": 8987, + "##ath": 8988, + "interchange": 8989, + "virtually": 8990, + "gen": 8991, + "gravity": 8992, + "exploration": 8993, + "amber": 8994, + "vital": 8995, + "wishes": 8996, + "powell": 8997, + "doctrine": 8998, + "elbow": 8999, + "screenplay": 9000, + "##bird": 9001, + "contribute": 9002, + "indonesian": 9003, + "pet": 9004, + "creates": 9005, + "##com": 9006, + "enzyme": 9007, + "kylie": 9008, + "discipline": 9009, + "drops": 9010, + "manila": 9011, + "hunger": 9012, + "##ien": 9013, + "layers": 9014, + "suffer": 9015, + "fever": 9016, + "bits": 9017, + "monica": 9018, + "keyboard": 9019, + "manages": 9020, + "##hood": 9021, + "searched": 9022, + "appeals": 9023, + "##bad": 9024, + "testament": 9025, + "grande": 9026, + "reid": 9027, + "##war": 9028, + "beliefs": 9029, + "congo": 9030, + "##ification": 9031, + "##dia": 9032, + "si": 9033, + "requiring": 9034, + "##via": 9035, + "casey": 9036, + "1849": 9037, + "regret": 9038, + "streak": 9039, + "rape": 9040, + "depends": 9041, + "syrian": 9042, + "sprint": 9043, + "pound": 9044, + "tourists": 9045, + "upcoming": 9046, + "pub": 9047, + "##xi": 9048, + "tense": 9049, + "##els": 9050, + "practiced": 9051, + "echo": 9052, + "nationwide": 9053, + "guild": 9054, + "motorcycle": 9055, + "liz": 9056, + "##zar": 9057, + "chiefs": 9058, + "desired": 9059, + "elena": 9060, + "bye": 9061, + "precious": 9062, + "absorbed": 9063, + "relatives": 9064, + "booth": 9065, + "pianist": 9066, + "##mal": 9067, + "citizenship": 9068, + "exhausted": 9069, + "wilhelm": 9070, + "##ceae": 9071, + "##hed": 9072, + "noting": 9073, + "quarterback": 9074, + "urge": 9075, + "hectares": 9076, + "##gue": 9077, + "ace": 9078, + "holly": 9079, + "##tal": 9080, + "blonde": 9081, + "davies": 9082, + "parked": 9083, + "sustainable": 9084, + "stepping": 9085, + "twentieth": 9086, + "airfield": 9087, + "galaxy": 9088, + "nest": 9089, + "chip": 9090, + "##nell": 9091, + "tan": 9092, + "shaft": 9093, + "paulo": 9094, + "requirement": 9095, + "##zy": 9096, + "paradise": 9097, + "tobacco": 9098, + "trans": 9099, + "renewed": 9100, + "vietnamese": 9101, + "##cker": 9102, + "##ju": 9103, + "suggesting": 9104, + "catching": 9105, + "holmes": 9106, + "enjoying": 9107, + "md": 9108, + "trips": 9109, + "colt": 9110, + "holder": 9111, + "butterfly": 9112, + "nerve": 9113, + "reformed": 9114, + "cherry": 9115, + "bowling": 9116, + "trailer": 9117, + "carriage": 9118, + "goodbye": 9119, + "appreciate": 9120, + "toy": 9121, + "joshua": 9122, + "interactive": 9123, + "enabled": 9124, + "involve": 9125, + "##kan": 9126, + "collar": 9127, + "determination": 9128, + "bunch": 9129, + "facebook": 9130, + "recall": 9131, + "shorts": 9132, + "superintendent": 9133, + "episcopal": 9134, + "frustration": 9135, + "giovanni": 9136, + "nineteenth": 9137, + "laser": 9138, + "privately": 9139, + "array": 9140, + "circulation": 9141, + "##ovic": 9142, + "armstrong": 9143, + "deals": 9144, + "painful": 9145, + "permit": 9146, + "discrimination": 9147, + "##wi": 9148, + "aires": 9149, + "retiring": 9150, + "cottage": 9151, + "ni": 9152, + "##sta": 9153, + "horizon": 9154, + "ellen": 9155, + "jamaica": 9156, + "ripped": 9157, + "fernando": 9158, + "chapters": 9159, + "playstation": 9160, + "patron": 9161, + "lecturer": 9162, + "navigation": 9163, + "behaviour": 9164, + "genes": 9165, + "georgian": 9166, + "export": 9167, + "solomon": 9168, + "rivals": 9169, + "swift": 9170, + "seventeen": 9171, + "rodriguez": 9172, + "princeton": 9173, + "independently": 9174, + "sox": 9175, + "1847": 9176, + "arguing": 9177, + "entity": 9178, + "casting": 9179, + "hank": 9180, + "criteria": 9181, + "oakland": 9182, + "geographic": 9183, + "milwaukee": 9184, + "reflection": 9185, + "expanding": 9186, + "conquest": 9187, + "dubbed": 9188, + "##tv": 9189, + "halt": 9190, + "brave": 9191, + "brunswick": 9192, + "doi": 9193, + "arched": 9194, + "curtis": 9195, + "divorced": 9196, + "predominantly": 9197, + "somerset": 9198, + "streams": 9199, + "ugly": 9200, + "zoo": 9201, + "horrible": 9202, + "curved": 9203, + "buenos": 9204, + "fierce": 9205, + "dictionary": 9206, + "vector": 9207, + "theological": 9208, + "unions": 9209, + "handful": 9210, + "stability": 9211, + "chan": 9212, + "punjab": 9213, + "segments": 9214, + "##lly": 9215, + "altar": 9216, + "ignoring": 9217, + "gesture": 9218, + "monsters": 9219, + "pastor": 9220, + "##stone": 9221, + "thighs": 9222, + "unexpected": 9223, + "operators": 9224, + "abruptly": 9225, + "coin": 9226, + "compiled": 9227, + "associates": 9228, + "improving": 9229, + "migration": 9230, + "pin": 9231, + "##ose": 9232, + "compact": 9233, + "collegiate": 9234, + "reserved": 9235, + "##urs": 9236, + "quarterfinals": 9237, + "roster": 9238, + "restore": 9239, + "assembled": 9240, + "hurry": 9241, + "oval": 9242, + "##cies": 9243, + "1846": 9244, + "flags": 9245, + "martha": 9246, + "##del": 9247, + "victories": 9248, + "sharply": 9249, + "##rated": 9250, + "argues": 9251, + "deadly": 9252, + "neo": 9253, + "drawings": 9254, + "symbols": 9255, + "performer": 9256, + "##iel": 9257, + "griffin": 9258, + "restrictions": 9259, + "editing": 9260, + "andrews": 9261, + "java": 9262, + "journals": 9263, + "arabia": 9264, + "compositions": 9265, + "dee": 9266, + "pierce": 9267, + "removing": 9268, + "hindi": 9269, + "casino": 9270, + "runway": 9271, + "civilians": 9272, + "minds": 9273, + "nasa": 9274, + "hotels": 9275, + "##zation": 9276, + "refuge": 9277, + "rent": 9278, + "retain": 9279, + "potentially": 9280, + "conferences": 9281, + "suburban": 9282, + "conducting": 9283, + "##tto": 9284, + "##tions": 9285, + "##tle": 9286, + "descended": 9287, + "massacre": 9288, + "##cal": 9289, + "ammunition": 9290, + "terrain": 9291, + "fork": 9292, + "souls": 9293, + "counts": 9294, + "chelsea": 9295, + "durham": 9296, + "drives": 9297, + "cab": 9298, + "##bank": 9299, + "perth": 9300, + "realizing": 9301, + "palestinian": 9302, + "finn": 9303, + "simpson": 9304, + "##dal": 9305, + "betty": 9306, + "##ule": 9307, + "moreover": 9308, + "particles": 9309, + "cardinals": 9310, + "tent": 9311, + "evaluation": 9312, + "extraordinary": 9313, + "##oid": 9314, + "inscription": 9315, + "##works": 9316, + "wednesday": 9317, + "chloe": 9318, + "maintains": 9319, + "panels": 9320, + "ashley": 9321, + "trucks": 9322, + "##nation": 9323, + "cluster": 9324, + "sunlight": 9325, + "strikes": 9326, + "zhang": 9327, + "##wing": 9328, + "dialect": 9329, + "canon": 9330, + "##ap": 9331, + "tucked": 9332, + "##ws": 9333, + "collecting": 9334, + "##mas": 9335, + "##can": 9336, + "##sville": 9337, + "maker": 9338, + "quoted": 9339, + "evan": 9340, + "franco": 9341, + "aria": 9342, + "buying": 9343, + "cleaning": 9344, + "eva": 9345, + "closet": 9346, + "provision": 9347, + "apollo": 9348, + "clinic": 9349, + "rat": 9350, + "##ez": 9351, + "necessarily": 9352, + "ac": 9353, + "##gle": 9354, + "##ising": 9355, + "venues": 9356, + "flipped": 9357, + "cent": 9358, + "spreading": 9359, + "trustees": 9360, + "checking": 9361, + "authorized": 9362, + "##sco": 9363, + "disappointed": 9364, + "##ado": 9365, + "notion": 9366, + "duration": 9367, + "trumpet": 9368, + "hesitated": 9369, + "topped": 9370, + "brussels": 9371, + "rolls": 9372, + "theoretical": 9373, + "hint": 9374, + "define": 9375, + "aggressive": 9376, + "repeat": 9377, + "wash": 9378, + "peaceful": 9379, + "optical": 9380, + "width": 9381, + "allegedly": 9382, + "mcdonald": 9383, + "strict": 9384, + "copyright": 9385, + "##illa": 9386, + "investors": 9387, + "mar": 9388, + "jam": 9389, + "witnesses": 9390, + "sounding": 9391, + "miranda": 9392, + "michelle": 9393, + "privacy": 9394, + "hugo": 9395, + "harmony": 9396, + "##pp": 9397, + "valid": 9398, + "lynn": 9399, + "glared": 9400, + "nina": 9401, + "102": 9402, + "headquartered": 9403, + "diving": 9404, + "boarding": 9405, + "gibson": 9406, + "##ncy": 9407, + "albanian": 9408, + "marsh": 9409, + "routine": 9410, + "dealt": 9411, + "enhanced": 9412, + "er": 9413, + "intelligent": 9414, + "substance": 9415, + "targeted": 9416, + "enlisted": 9417, + "discovers": 9418, + "spinning": 9419, + "observations": 9420, + "pissed": 9421, + "smoking": 9422, + "rebecca": 9423, + "capitol": 9424, + "visa": 9425, + "varied": 9426, + "costume": 9427, + "seemingly": 9428, + "indies": 9429, + "compensation": 9430, + "surgeon": 9431, + "thursday": 9432, + "arsenal": 9433, + "westminster": 9434, + "suburbs": 9435, + "rid": 9436, + "anglican": 9437, + "##ridge": 9438, + "knots": 9439, + "foods": 9440, + "alumni": 9441, + "lighter": 9442, + "fraser": 9443, + "whoever": 9444, + "portal": 9445, + "scandal": 9446, + "##ray": 9447, + "gavin": 9448, + "advised": 9449, + "instructor": 9450, + "flooding": 9451, + "terrorist": 9452, + "##ale": 9453, + "teenage": 9454, + "interim": 9455, + "senses": 9456, + "duck": 9457, + "teen": 9458, + "thesis": 9459, + "abby": 9460, + "eager": 9461, + "overcome": 9462, + "##ile": 9463, + "newport": 9464, + "glenn": 9465, + "rises": 9466, + "shame": 9467, + "##cc": 9468, + "prompted": 9469, + "priority": 9470, + "forgot": 9471, + "bomber": 9472, + "nicolas": 9473, + "protective": 9474, + "360": 9475, + "cartoon": 9476, + "katherine": 9477, + "breeze": 9478, + "lonely": 9479, + "trusted": 9480, + "henderson": 9481, + "richardson": 9482, + "relax": 9483, + "banner": 9484, + "candy": 9485, + "palms": 9486, + "remarkable": 9487, + "##rio": 9488, + "legends": 9489, + "cricketer": 9490, + "essay": 9491, + "ordained": 9492, + "edmund": 9493, + "rifles": 9494, + "trigger": 9495, + "##uri": 9496, + "##away": 9497, + "sail": 9498, + "alert": 9499, + "1830": 9500, + "audiences": 9501, + "penn": 9502, + "sussex": 9503, + "siblings": 9504, + "pursued": 9505, + "indianapolis": 9506, + "resist": 9507, + "rosa": 9508, + "consequence": 9509, + "succeed": 9510, + "avoided": 9511, + "1845": 9512, + "##ulation": 9513, + "inland": 9514, + "##tie": 9515, + "##nna": 9516, + "counsel": 9517, + "profession": 9518, + "chronicle": 9519, + "hurried": 9520, + "##una": 9521, + "eyebrow": 9522, + "eventual": 9523, + "bleeding": 9524, + "innovative": 9525, + "cure": 9526, + "##dom": 9527, + "committees": 9528, + "accounting": 9529, + "con": 9530, + "scope": 9531, + "hardy": 9532, + "heather": 9533, + "tenor": 9534, + "gut": 9535, + "herald": 9536, + "codes": 9537, + "tore": 9538, + "scales": 9539, + "wagon": 9540, + "##oo": 9541, + "luxury": 9542, + "tin": 9543, + "prefer": 9544, + "fountain": 9545, + "triangle": 9546, + "bonds": 9547, + "darling": 9548, + "convoy": 9549, + "dried": 9550, + "traced": 9551, + "beings": 9552, + "troy": 9553, + "accidentally": 9554, + "slam": 9555, + "findings": 9556, + "smelled": 9557, + "joey": 9558, + "lawyers": 9559, + "outcome": 9560, + "steep": 9561, + "bosnia": 9562, + "configuration": 9563, + "shifting": 9564, + "toll": 9565, + "brook": 9566, + "performers": 9567, + "lobby": 9568, + "philosophical": 9569, + "construct": 9570, + "shrine": 9571, + "aggregate": 9572, + "boot": 9573, + "cox": 9574, + "phenomenon": 9575, + "savage": 9576, + "insane": 9577, + "solely": 9578, + "reynolds": 9579, + "lifestyle": 9580, + "##ima": 9581, + "nationally": 9582, + "holdings": 9583, + "consideration": 9584, + "enable": 9585, + "edgar": 9586, + "mo": 9587, + "mama": 9588, + "##tein": 9589, + "fights": 9590, + "relegation": 9591, + "chances": 9592, + "atomic": 9593, + "hub": 9594, + "conjunction": 9595, + "awkward": 9596, + "reactions": 9597, + "currency": 9598, + "finale": 9599, + "kumar": 9600, + "underwent": 9601, + "steering": 9602, + "elaborate": 9603, + "gifts": 9604, + "comprising": 9605, + "melissa": 9606, + "veins": 9607, + "reasonable": 9608, + "sunshine": 9609, + "chi": 9610, + "solve": 9611, + "trails": 9612, + "inhabited": 9613, + "elimination": 9614, + "ethics": 9615, + "huh": 9616, + "ana": 9617, + "molly": 9618, + "consent": 9619, + "apartments": 9620, + "layout": 9621, + "marines": 9622, + "##ces": 9623, + "hunters": 9624, + "bulk": 9625, + "##oma": 9626, + "hometown": 9627, + "##wall": 9628, + "##mont": 9629, + "cracked": 9630, + "reads": 9631, + "neighbouring": 9632, + "withdrawn": 9633, + "admission": 9634, + "wingspan": 9635, + "damned": 9636, + "anthology": 9637, + "lancashire": 9638, + "brands": 9639, + "batting": 9640, + "forgive": 9641, + "cuban": 9642, + "awful": 9643, + "##lyn": 9644, + "104": 9645, + "dimensions": 9646, + "imagination": 9647, + "##ade": 9648, + "dante": 9649, + "##ship": 9650, + "tracking": 9651, + "desperately": 9652, + "goalkeeper": 9653, + "##yne": 9654, + "groaned": 9655, + "workshops": 9656, + "confident": 9657, + "burton": 9658, + "gerald": 9659, + "milton": 9660, + "circus": 9661, + "uncertain": 9662, + "slope": 9663, + "copenhagen": 9664, + "sophia": 9665, + "fog": 9666, + "philosopher": 9667, + "portraits": 9668, + "accent": 9669, + "cycling": 9670, + "varying": 9671, + "gripped": 9672, + "larvae": 9673, + "garrett": 9674, + "specified": 9675, + "scotia": 9676, + "mature": 9677, + "luther": 9678, + "kurt": 9679, + "rap": 9680, + "##kes": 9681, + "aerial": 9682, + "750": 9683, + "ferdinand": 9684, + "heated": 9685, + "es": 9686, + "transported": 9687, + "##shan": 9688, + "safely": 9689, + "nonetheless": 9690, + "##orn": 9691, + "##gal": 9692, + "motors": 9693, + "demanding": 9694, + "##sburg": 9695, + "startled": 9696, + "##brook": 9697, + "ally": 9698, + "generate": 9699, + "caps": 9700, + "ghana": 9701, + "stained": 9702, + "demo": 9703, + "mentions": 9704, + "beds": 9705, + "ap": 9706, + "afterward": 9707, + "diary": 9708, + "##bling": 9709, + "utility": 9710, + "##iro": 9711, + "richards": 9712, + "1837": 9713, + "conspiracy": 9714, + "conscious": 9715, + "shining": 9716, + "footsteps": 9717, + "observer": 9718, + "cyprus": 9719, + "urged": 9720, + "loyalty": 9721, + "developer": 9722, + "probability": 9723, + "olive": 9724, + "upgraded": 9725, + "gym": 9726, + "miracle": 9727, + "insects": 9728, + "graves": 9729, + "1844": 9730, + "ourselves": 9731, + "hydrogen": 9732, + "amazon": 9733, + "katie": 9734, + "tickets": 9735, + "poets": 9736, + "##pm": 9737, + "planes": 9738, + "##pan": 9739, + "prevention": 9740, + "witnessed": 9741, + "dense": 9742, + "jin": 9743, + "randy": 9744, + "tang": 9745, + "warehouse": 9746, + "monroe": 9747, + "bang": 9748, + "archived": 9749, + "elderly": 9750, + "investigations": 9751, + "alec": 9752, + "granite": 9753, + "mineral": 9754, + "conflicts": 9755, + "controlling": 9756, + "aboriginal": 9757, + "carlo": 9758, + "##zu": 9759, + "mechanics": 9760, + "stan": 9761, + "stark": 9762, + "rhode": 9763, + "skirt": 9764, + "est": 9765, + "##berry": 9766, + "bombs": 9767, + "respected": 9768, + "##horn": 9769, + "imposed": 9770, + "limestone": 9771, + "deny": 9772, + "nominee": 9773, + "memphis": 9774, + "grabbing": 9775, + "disabled": 9776, + "##als": 9777, + "amusement": 9778, + "aa": 9779, + "frankfurt": 9780, + "corn": 9781, + "referendum": 9782, + "varies": 9783, + "slowed": 9784, + "disk": 9785, + "firms": 9786, + "unconscious": 9787, + "incredible": 9788, + "clue": 9789, + "sue": 9790, + "##zhou": 9791, + "twist": 9792, + "##cio": 9793, + "joins": 9794, + "idaho": 9795, + "chad": 9796, + "developers": 9797, + "computing": 9798, + "destroyer": 9799, + "103": 9800, + "mortal": 9801, + "tucker": 9802, + "kingston": 9803, + "choices": 9804, + "yu": 9805, + "carson": 9806, + "1800": 9807, + "os": 9808, + "whitney": 9809, + "geneva": 9810, + "pretend": 9811, + "dimension": 9812, + "staged": 9813, + "plateau": 9814, + "maya": 9815, + "##une": 9816, + "freestyle": 9817, + "##bc": 9818, + "rovers": 9819, + "hiv": 9820, + "##ids": 9821, + "tristan": 9822, + "classroom": 9823, + "prospect": 9824, + "##hus": 9825, + "honestly": 9826, + "diploma": 9827, + "lied": 9828, + "thermal": 9829, + "auxiliary": 9830, + "feast": 9831, + "unlikely": 9832, + "iata": 9833, + "##tel": 9834, + "morocco": 9835, + "pounding": 9836, + "treasury": 9837, + "lithuania": 9838, + "considerably": 9839, + "1841": 9840, + "dish": 9841, + "1812": 9842, + "geological": 9843, + "matching": 9844, + "stumbled": 9845, + "destroying": 9846, + "marched": 9847, + "brien": 9848, + "advances": 9849, + "cake": 9850, + "nicole": 9851, + "belle": 9852, + "settling": 9853, + "measuring": 9854, + "directing": 9855, + "##mie": 9856, + "tuesday": 9857, + "bassist": 9858, + "capabilities": 9859, + "stunned": 9860, + "fraud": 9861, + "torpedo": 9862, + "##list": 9863, + "##phone": 9864, + "anton": 9865, + "wisdom": 9866, + "surveillance": 9867, + "ruined": 9868, + "##ulate": 9869, + "lawsuit": 9870, + "healthcare": 9871, + "theorem": 9872, + "halls": 9873, + "trend": 9874, + "aka": 9875, + "horizontal": 9876, + "dozens": 9877, + "acquire": 9878, + "lasting": 9879, + "swim": 9880, + "hawk": 9881, + "gorgeous": 9882, + "fees": 9883, + "vicinity": 9884, + "decrease": 9885, + "adoption": 9886, + "tactics": 9887, + "##ography": 9888, + "pakistani": 9889, + "##ole": 9890, + "draws": 9891, + "##hall": 9892, + "willie": 9893, + "burke": 9894, + "heath": 9895, + "algorithm": 9896, + "integral": 9897, + "powder": 9898, + "elliott": 9899, + "brigadier": 9900, + "jackie": 9901, + "tate": 9902, + "varieties": 9903, + "darker": 9904, + "##cho": 9905, + "lately": 9906, + "cigarette": 9907, + "specimens": 9908, + "adds": 9909, + "##ree": 9910, + "##ensis": 9911, + "##inger": 9912, + "exploded": 9913, + "finalist": 9914, + "cia": 9915, + "murders": 9916, + "wilderness": 9917, + "arguments": 9918, + "nicknamed": 9919, + "acceptance": 9920, + "onwards": 9921, + "manufacture": 9922, + "robertson": 9923, + "jets": 9924, + "tampa": 9925, + "enterprises": 9926, + "blog": 9927, + "loudly": 9928, + "composers": 9929, + "nominations": 9930, + "1838": 9931, + "ai": 9932, + "malta": 9933, + "inquiry": 9934, + "automobile": 9935, + "hosting": 9936, + "viii": 9937, + "rays": 9938, + "tilted": 9939, + "grief": 9940, + "museums": 9941, + "strategies": 9942, + "furious": 9943, + "euro": 9944, + "equality": 9945, + "cohen": 9946, + "poison": 9947, + "surrey": 9948, + "wireless": 9949, + "governed": 9950, + "ridiculous": 9951, + "moses": 9952, + "##esh": 9953, + "##room": 9954, + "vanished": 9955, + "##ito": 9956, + "barnes": 9957, + "attract": 9958, + "morrison": 9959, + "istanbul": 9960, + "##iness": 9961, + "absent": 9962, + "rotation": 9963, + "petition": 9964, + "janet": 9965, + "##logical": 9966, + "satisfaction": 9967, + "custody": 9968, + "deliberately": 9969, + "observatory": 9970, + "comedian": 9971, + "surfaces": 9972, + "pinyin": 9973, + "novelist": 9974, + "strictly": 9975, + "canterbury": 9976, + "oslo": 9977, + "monks": 9978, + "embrace": 9979, + "ibm": 9980, + "jealous": 9981, + "photograph": 9982, + "continent": 9983, + "dorothy": 9984, + "marina": 9985, + "doc": 9986, + "excess": 9987, + "holden": 9988, + "allegations": 9989, + "explaining": 9990, + "stack": 9991, + "avoiding": 9992, + "lance": 9993, + "storyline": 9994, + "majesty": 9995, + "poorly": 9996, + "spike": 9997, + "dos": 9998, + "bradford": 9999, + "raven": 10000, + "travis": 10001, + "classics": 10002, + "proven": 10003, + "voltage": 10004, + "pillow": 10005, + "fists": 10006, + "butt": 10007, + "1842": 10008, + "interpreted": 10009, + "##car": 10010, + "1839": 10011, + "gage": 10012, + "telegraph": 10013, + "lens": 10014, + "promising": 10015, + "expelled": 10016, + "casual": 10017, + "collector": 10018, + "zones": 10019, + "##min": 10020, + "silly": 10021, + "nintendo": 10022, + "##kh": 10023, + "##bra": 10024, + "downstairs": 10025, + "chef": 10026, + "suspicious": 10027, + "afl": 10028, + "flies": 10029, + "vacant": 10030, + "uganda": 10031, + "pregnancy": 10032, + "condemned": 10033, + "lutheran": 10034, + "estimates": 10035, + "cheap": 10036, + "decree": 10037, + "saxon": 10038, + "proximity": 10039, + "stripped": 10040, + "idiot": 10041, + "deposits": 10042, + "contrary": 10043, + "presenter": 10044, + "magnus": 10045, + "glacier": 10046, + "im": 10047, + "offense": 10048, + "edwin": 10049, + "##ori": 10050, + "upright": 10051, + "##long": 10052, + "bolt": 10053, + "##ois": 10054, + "toss": 10055, + "geographical": 10056, + "##izes": 10057, + "environments": 10058, + "delicate": 10059, + "marking": 10060, + "abstract": 10061, + "xavier": 10062, + "nails": 10063, + "windsor": 10064, + "plantation": 10065, + "occurring": 10066, + "equity": 10067, + "saskatchewan": 10068, + "fears": 10069, + "drifted": 10070, + "sequences": 10071, + "vegetation": 10072, + "revolt": 10073, + "##stic": 10074, + "1843": 10075, + "sooner": 10076, + "fusion": 10077, + "opposing": 10078, + "nato": 10079, + "skating": 10080, + "1836": 10081, + "secretly": 10082, + "ruin": 10083, + "lease": 10084, + "##oc": 10085, + "edit": 10086, + "##nne": 10087, + "flora": 10088, + "anxiety": 10089, + "ruby": 10090, + "##ological": 10091, + "##mia": 10092, + "tel": 10093, + "bout": 10094, + "taxi": 10095, + "emmy": 10096, + "frost": 10097, + "rainbow": 10098, + "compounds": 10099, + "foundations": 10100, + "rainfall": 10101, + "assassination": 10102, + "nightmare": 10103, + "dominican": 10104, + "##win": 10105, + "achievements": 10106, + "deserve": 10107, + "orlando": 10108, + "intact": 10109, + "armenia": 10110, + "##nte": 10111, + "calgary": 10112, + "valentine": 10113, + "106": 10114, + "marion": 10115, + "proclaimed": 10116, + "theodore": 10117, + "bells": 10118, + "courtyard": 10119, + "thigh": 10120, + "gonzalez": 10121, + "console": 10122, + "troop": 10123, + "minimal": 10124, + "monte": 10125, + "everyday": 10126, + "##ence": 10127, + "##if": 10128, + "supporter": 10129, + "terrorism": 10130, + "buck": 10131, + "openly": 10132, + "presbyterian": 10133, + "activists": 10134, + "carpet": 10135, + "##iers": 10136, + "rubbing": 10137, + "uprising": 10138, + "##yi": 10139, + "cute": 10140, + "conceived": 10141, + "legally": 10142, + "##cht": 10143, + "millennium": 10144, + "cello": 10145, + "velocity": 10146, + "ji": 10147, + "rescued": 10148, + "cardiff": 10149, + "1835": 10150, + "rex": 10151, + "concentrate": 10152, + "senators": 10153, + "beard": 10154, + "rendered": 10155, + "glowing": 10156, + "battalions": 10157, + "scouts": 10158, + "competitors": 10159, + "sculptor": 10160, + "catalogue": 10161, + "arctic": 10162, + "ion": 10163, + "raja": 10164, + "bicycle": 10165, + "wow": 10166, + "glancing": 10167, + "lawn": 10168, + "##woman": 10169, + "gentleman": 10170, + "lighthouse": 10171, + "publish": 10172, + "predicted": 10173, + "calculated": 10174, + "##val": 10175, + "variants": 10176, + "##gne": 10177, + "strain": 10178, + "##ui": 10179, + "winston": 10180, + "deceased": 10181, + "##nus": 10182, + "touchdowns": 10183, + "brady": 10184, + "caleb": 10185, + "sinking": 10186, + "echoed": 10187, + "crush": 10188, + "hon": 10189, + "blessed": 10190, + "protagonist": 10191, + "hayes": 10192, + "endangered": 10193, + "magnitude": 10194, + "editors": 10195, + "##tine": 10196, + "estimate": 10197, + "responsibilities": 10198, + "##mel": 10199, + "backup": 10200, + "laying": 10201, + "consumed": 10202, + "sealed": 10203, + "zurich": 10204, + "lovers": 10205, + "frustrated": 10206, + "##eau": 10207, + "ahmed": 10208, + "kicking": 10209, + "mit": 10210, + "treasurer": 10211, + "1832": 10212, + "biblical": 10213, + "refuse": 10214, + "terrified": 10215, + "pump": 10216, + "agrees": 10217, + "genuine": 10218, + "imprisonment": 10219, + "refuses": 10220, + "plymouth": 10221, + "##hen": 10222, + "lou": 10223, + "##nen": 10224, + "tara": 10225, + "trembling": 10226, + "antarctic": 10227, + "ton": 10228, + "learns": 10229, + "##tas": 10230, + "crap": 10231, + "crucial": 10232, + "faction": 10233, + "atop": 10234, + "##borough": 10235, + "wrap": 10236, + "lancaster": 10237, + "odds": 10238, + "hopkins": 10239, + "erik": 10240, + "lyon": 10241, + "##eon": 10242, + "bros": 10243, + "##ode": 10244, + "snap": 10245, + "locality": 10246, + "tips": 10247, + "empress": 10248, + "crowned": 10249, + "cal": 10250, + "acclaimed": 10251, + "chuckled": 10252, + "##ory": 10253, + "clara": 10254, + "sends": 10255, + "mild": 10256, + "towel": 10257, + "##fl": 10258, + "##day": 10259, + "##а": 10260, + "wishing": 10261, + "assuming": 10262, + "interviewed": 10263, + "##bal": 10264, + "##die": 10265, + "interactions": 10266, + "eden": 10267, + "cups": 10268, + "helena": 10269, + "##lf": 10270, + "indie": 10271, + "beck": 10272, + "##fire": 10273, + "batteries": 10274, + "filipino": 10275, + "wizard": 10276, + "parted": 10277, + "##lam": 10278, + "traces": 10279, + "##born": 10280, + "rows": 10281, + "idol": 10282, + "albany": 10283, + "delegates": 10284, + "##ees": 10285, + "##sar": 10286, + "discussions": 10287, + "##ex": 10288, + "notre": 10289, + "instructed": 10290, + "belgrade": 10291, + "highways": 10292, + "suggestion": 10293, + "lauren": 10294, + "possess": 10295, + "orientation": 10296, + "alexandria": 10297, + "abdul": 10298, + "beats": 10299, + "salary": 10300, + "reunion": 10301, + "ludwig": 10302, + "alright": 10303, + "wagner": 10304, + "intimate": 10305, + "pockets": 10306, + "slovenia": 10307, + "hugged": 10308, + "brighton": 10309, + "merchants": 10310, + "cruel": 10311, + "stole": 10312, + "trek": 10313, + "slopes": 10314, + "repairs": 10315, + "enrollment": 10316, + "politically": 10317, + "underlying": 10318, + "promotional": 10319, + "counting": 10320, + "boeing": 10321, + "##bb": 10322, + "isabella": 10323, + "naming": 10324, + "##и": 10325, + "keen": 10326, + "bacteria": 10327, + "listing": 10328, + "separately": 10329, + "belfast": 10330, + "ussr": 10331, + "450": 10332, + "lithuanian": 10333, + "anybody": 10334, + "ribs": 10335, + "sphere": 10336, + "martinez": 10337, + "cock": 10338, + "embarrassed": 10339, + "proposals": 10340, + "fragments": 10341, + "nationals": 10342, + "##fs": 10343, + "##wski": 10344, + "premises": 10345, + "fin": 10346, + "1500": 10347, + "alpine": 10348, + "matched": 10349, + "freely": 10350, + "bounded": 10351, + "jace": 10352, + "sleeve": 10353, + "##af": 10354, + "gaming": 10355, + "pier": 10356, + "populated": 10357, + "evident": 10358, + "##like": 10359, + "frances": 10360, + "flooded": 10361, + "##dle": 10362, + "frightened": 10363, + "pour": 10364, + "trainer": 10365, + "framed": 10366, + "visitor": 10367, + "challenging": 10368, + "pig": 10369, + "wickets": 10370, + "##fold": 10371, + "infected": 10372, + "email": 10373, + "##pes": 10374, + "arose": 10375, + "##aw": 10376, + "reward": 10377, + "ecuador": 10378, + "oblast": 10379, + "vale": 10380, + "ch": 10381, + "shuttle": 10382, + "##usa": 10383, + "bach": 10384, + "rankings": 10385, + "forbidden": 10386, + "cornwall": 10387, + "accordance": 10388, + "salem": 10389, + "consumers": 10390, + "bruno": 10391, + "fantastic": 10392, + "toes": 10393, + "machinery": 10394, + "resolved": 10395, + "julius": 10396, + "remembering": 10397, + "propaganda": 10398, + "iceland": 10399, + "bombardment": 10400, + "tide": 10401, + "contacts": 10402, + "wives": 10403, + "##rah": 10404, + "concerto": 10405, + "macdonald": 10406, + "albania": 10407, + "implement": 10408, + "daisy": 10409, + "tapped": 10410, + "sudan": 10411, + "helmet": 10412, + "angela": 10413, + "mistress": 10414, + "##lic": 10415, + "crop": 10416, + "sunk": 10417, + "finest": 10418, + "##craft": 10419, + "hostile": 10420, + "##ute": 10421, + "##tsu": 10422, + "boxer": 10423, + "fr": 10424, + "paths": 10425, + "adjusted": 10426, + "habit": 10427, + "ballot": 10428, + "supervision": 10429, + "soprano": 10430, + "##zen": 10431, + "bullets": 10432, + "wicked": 10433, + "sunset": 10434, + "regiments": 10435, + "disappear": 10436, + "lamp": 10437, + "performs": 10438, + "app": 10439, + "##gia": 10440, + "##oa": 10441, + "rabbit": 10442, + "digging": 10443, + "incidents": 10444, + "entries": 10445, + "##cion": 10446, + "dishes": 10447, + "##oi": 10448, + "introducing": 10449, + "##ati": 10450, + "##fied": 10451, + "freshman": 10452, + "slot": 10453, + "jill": 10454, + "tackles": 10455, + "baroque": 10456, + "backs": 10457, + "##iest": 10458, + "lone": 10459, + "sponsor": 10460, + "destiny": 10461, + "altogether": 10462, + "convert": 10463, + "##aro": 10464, + "consensus": 10465, + "shapes": 10466, + "demonstration": 10467, + "basically": 10468, + "feminist": 10469, + "auction": 10470, + "artifacts": 10471, + "##bing": 10472, + "strongest": 10473, + "twitter": 10474, + "halifax": 10475, + "2019": 10476, + "allmusic": 10477, + "mighty": 10478, + "smallest": 10479, + "precise": 10480, + "alexandra": 10481, + "viola": 10482, + "##los": 10483, + "##ille": 10484, + "manuscripts": 10485, + "##illo": 10486, + "dancers": 10487, + "ari": 10488, + "managers": 10489, + "monuments": 10490, + "blades": 10491, + "barracks": 10492, + "springfield": 10493, + "maiden": 10494, + "consolidated": 10495, + "electron": 10496, + "##end": 10497, + "berry": 10498, + "airing": 10499, + "wheat": 10500, + "nobel": 10501, + "inclusion": 10502, + "blair": 10503, + "payments": 10504, + "geography": 10505, + "bee": 10506, + "cc": 10507, + "eleanor": 10508, + "react": 10509, + "##hurst": 10510, + "afc": 10511, + "manitoba": 10512, + "##yu": 10513, + "su": 10514, + "lineup": 10515, + "fitness": 10516, + "recreational": 10517, + "investments": 10518, + "airborne": 10519, + "disappointment": 10520, + "##dis": 10521, + "edmonton": 10522, + "viewing": 10523, + "##row": 10524, + "renovation": 10525, + "##cast": 10526, + "infant": 10527, + "bankruptcy": 10528, + "roses": 10529, + "aftermath": 10530, + "pavilion": 10531, + "##yer": 10532, + "carpenter": 10533, + "withdrawal": 10534, + "ladder": 10535, + "##hy": 10536, + "discussing": 10537, + "popped": 10538, + "reliable": 10539, + "agreements": 10540, + "rochester": 10541, + "##abad": 10542, + "curves": 10543, + "bombers": 10544, + "220": 10545, + "rao": 10546, + "reverend": 10547, + "decreased": 10548, + "choosing": 10549, + "107": 10550, + "stiff": 10551, + "consulting": 10552, + "naples": 10553, + "crawford": 10554, + "tracy": 10555, + "ka": 10556, + "ribbon": 10557, + "cops": 10558, + "##lee": 10559, + "crushed": 10560, + "deciding": 10561, + "unified": 10562, + "teenager": 10563, + "accepting": 10564, + "flagship": 10565, + "explorer": 10566, + "poles": 10567, + "sanchez": 10568, + "inspection": 10569, + "revived": 10570, + "skilled": 10571, + "induced": 10572, + "exchanged": 10573, + "flee": 10574, + "locals": 10575, + "tragedy": 10576, + "swallow": 10577, + "loading": 10578, + "hanna": 10579, + "demonstrate": 10580, + "##ela": 10581, + "salvador": 10582, + "flown": 10583, + "contestants": 10584, + "civilization": 10585, + "##ines": 10586, + "wanna": 10587, + "rhodes": 10588, + "fletcher": 10589, + "hector": 10590, + "knocking": 10591, + "considers": 10592, + "##ough": 10593, + "nash": 10594, + "mechanisms": 10595, + "sensed": 10596, + "mentally": 10597, + "walt": 10598, + "unclear": 10599, + "##eus": 10600, + "renovated": 10601, + "madame": 10602, + "##cks": 10603, + "crews": 10604, + "governmental": 10605, + "##hin": 10606, + "undertaken": 10607, + "monkey": 10608, + "##ben": 10609, + "##ato": 10610, + "fatal": 10611, + "armored": 10612, + "copa": 10613, + "caves": 10614, + "governance": 10615, + "grasp": 10616, + "perception": 10617, + "certification": 10618, + "froze": 10619, + "damp": 10620, + "tugged": 10621, + "wyoming": 10622, + "##rg": 10623, + "##ero": 10624, + "newman": 10625, + "##lor": 10626, + "nerves": 10627, + "curiosity": 10628, + "graph": 10629, + "115": 10630, + "##ami": 10631, + "withdraw": 10632, + "tunnels": 10633, + "dull": 10634, + "meredith": 10635, + "moss": 10636, + "exhibits": 10637, + "neighbors": 10638, + "communicate": 10639, + "accuracy": 10640, + "explored": 10641, + "raiders": 10642, + "republicans": 10643, + "secular": 10644, + "kat": 10645, + "superman": 10646, + "penny": 10647, + "criticised": 10648, + "##tch": 10649, + "freed": 10650, + "update": 10651, + "conviction": 10652, + "wade": 10653, + "ham": 10654, + "likewise": 10655, + "delegation": 10656, + "gotta": 10657, + "doll": 10658, + "promises": 10659, + "technological": 10660, + "myth": 10661, + "nationality": 10662, + "resolve": 10663, + "convent": 10664, + "##mark": 10665, + "sharon": 10666, + "dig": 10667, + "sip": 10668, + "coordinator": 10669, + "entrepreneur": 10670, + "fold": 10671, + "##dine": 10672, + "capability": 10673, + "councillor": 10674, + "synonym": 10675, + "blown": 10676, + "swan": 10677, + "cursed": 10678, + "1815": 10679, + "jonas": 10680, + "haired": 10681, + "sofa": 10682, + "canvas": 10683, + "keeper": 10684, + "rivalry": 10685, + "##hart": 10686, + "rapper": 10687, + "speedway": 10688, + "swords": 10689, + "postal": 10690, + "maxwell": 10691, + "estonia": 10692, + "potter": 10693, + "recurring": 10694, + "##nn": 10695, + "##ave": 10696, + "errors": 10697, + "##oni": 10698, + "cognitive": 10699, + "1834": 10700, + "##²": 10701, + "claws": 10702, + "nadu": 10703, + "roberto": 10704, + "bce": 10705, + "wrestler": 10706, + "ellie": 10707, + "##ations": 10708, + "infinite": 10709, + "ink": 10710, + "##tia": 10711, + "presumably": 10712, + "finite": 10713, + "staircase": 10714, + "108": 10715, + "noel": 10716, + "patricia": 10717, + "nacional": 10718, + "##cation": 10719, + "chill": 10720, + "eternal": 10721, + "tu": 10722, + "preventing": 10723, + "prussia": 10724, + "fossil": 10725, + "limbs": 10726, + "##logist": 10727, + "ernst": 10728, + "frog": 10729, + "perez": 10730, + "rene": 10731, + "##ace": 10732, + "pizza": 10733, + "prussian": 10734, + "##ios": 10735, + "##vy": 10736, + "molecules": 10737, + "regulatory": 10738, + "answering": 10739, + "opinions": 10740, + "sworn": 10741, + "lengths": 10742, + "supposedly": 10743, + "hypothesis": 10744, + "upward": 10745, + "habitats": 10746, + "seating": 10747, + "ancestors": 10748, + "drank": 10749, + "yield": 10750, + "hd": 10751, + "synthesis": 10752, + "researcher": 10753, + "modest": 10754, + "##var": 10755, + "mothers": 10756, + "peered": 10757, + "voluntary": 10758, + "homeland": 10759, + "##the": 10760, + "acclaim": 10761, + "##igan": 10762, + "static": 10763, + "valve": 10764, + "luxembourg": 10765, + "alto": 10766, + "carroll": 10767, + "fe": 10768, + "receptor": 10769, + "norton": 10770, + "ambulance": 10771, + "##tian": 10772, + "johnston": 10773, + "catholics": 10774, + "depicting": 10775, + "jointly": 10776, + "elephant": 10777, + "gloria": 10778, + "mentor": 10779, + "badge": 10780, + "ahmad": 10781, + "distinguish": 10782, + "remarked": 10783, + "councils": 10784, + "precisely": 10785, + "allison": 10786, + "advancing": 10787, + "detection": 10788, + "crowded": 10789, + "##10": 10790, + "cooperative": 10791, + "ankle": 10792, + "mercedes": 10793, + "dagger": 10794, + "surrendered": 10795, + "pollution": 10796, + "commit": 10797, + "subway": 10798, + "jeffrey": 10799, + "lesson": 10800, + "sculptures": 10801, + "provider": 10802, + "##fication": 10803, + "membrane": 10804, + "timothy": 10805, + "rectangular": 10806, + "fiscal": 10807, + "heating": 10808, + "teammate": 10809, + "basket": 10810, + "particle": 10811, + "anonymous": 10812, + "deployment": 10813, + "##ple": 10814, + "missiles": 10815, + "courthouse": 10816, + "proportion": 10817, + "shoe": 10818, + "sec": 10819, + "##ller": 10820, + "complaints": 10821, + "forbes": 10822, + "blacks": 10823, + "abandon": 10824, + "remind": 10825, + "sizes": 10826, + "overwhelming": 10827, + "autobiography": 10828, + "natalie": 10829, + "##awa": 10830, + "risks": 10831, + "contestant": 10832, + "countryside": 10833, + "babies": 10834, + "scorer": 10835, + "invaded": 10836, + "enclosed": 10837, + "proceed": 10838, + "hurling": 10839, + "disorders": 10840, + "##cu": 10841, + "reflecting": 10842, + "continuously": 10843, + "cruiser": 10844, + "graduates": 10845, + "freeway": 10846, + "investigated": 10847, + "ore": 10848, + "deserved": 10849, + "maid": 10850, + "blocking": 10851, + "phillip": 10852, + "jorge": 10853, + "shakes": 10854, + "dove": 10855, + "mann": 10856, + "variables": 10857, + "lacked": 10858, + "burden": 10859, + "accompanying": 10860, + "que": 10861, + "consistently": 10862, + "organizing": 10863, + "provisional": 10864, + "complained": 10865, + "endless": 10866, + "##rm": 10867, + "tubes": 10868, + "juice": 10869, + "georges": 10870, + "krishna": 10871, + "mick": 10872, + "labels": 10873, + "thriller": 10874, + "##uch": 10875, + "laps": 10876, + "arcade": 10877, + "sage": 10878, + "snail": 10879, + "##table": 10880, + "shannon": 10881, + "fi": 10882, + "laurence": 10883, + "seoul": 10884, + "vacation": 10885, + "presenting": 10886, + "hire": 10887, + "churchill": 10888, + "surprisingly": 10889, + "prohibited": 10890, + "savannah": 10891, + "technically": 10892, + "##oli": 10893, + "170": 10894, + "##lessly": 10895, + "testimony": 10896, + "suited": 10897, + "speeds": 10898, + "toys": 10899, + "romans": 10900, + "mlb": 10901, + "flowering": 10902, + "measurement": 10903, + "talented": 10904, + "kay": 10905, + "settings": 10906, + "charleston": 10907, + "expectations": 10908, + "shattered": 10909, + "achieving": 10910, + "triumph": 10911, + "ceremonies": 10912, + "portsmouth": 10913, + "lanes": 10914, + "mandatory": 10915, + "loser": 10916, + "stretching": 10917, + "cologne": 10918, + "realizes": 10919, + "seventy": 10920, + "cornell": 10921, + "careers": 10922, + "webb": 10923, + "##ulating": 10924, + "americas": 10925, + "budapest": 10926, + "ava": 10927, + "suspicion": 10928, + "##ison": 10929, + "yo": 10930, + "conrad": 10931, + "##hai": 10932, + "sterling": 10933, + "jessie": 10934, + "rector": 10935, + "##az": 10936, + "1831": 10937, + "transform": 10938, + "organize": 10939, + "loans": 10940, + "christine": 10941, + "volcanic": 10942, + "warrant": 10943, + "slender": 10944, + "summers": 10945, + "subfamily": 10946, + "newer": 10947, + "danced": 10948, + "dynamics": 10949, + "rhine": 10950, + "proceeds": 10951, + "heinrich": 10952, + "gastropod": 10953, + "commands": 10954, + "sings": 10955, + "facilitate": 10956, + "easter": 10957, + "ra": 10958, + "positioned": 10959, + "responses": 10960, + "expense": 10961, + "fruits": 10962, + "yanked": 10963, + "imported": 10964, + "25th": 10965, + "velvet": 10966, + "vic": 10967, + "primitive": 10968, + "tribune": 10969, + "baldwin": 10970, + "neighbourhood": 10971, + "donna": 10972, + "rip": 10973, + "hay": 10974, + "pr": 10975, + "##uro": 10976, + "1814": 10977, + "espn": 10978, + "welcomed": 10979, + "##aria": 10980, + "qualifier": 10981, + "glare": 10982, + "highland": 10983, + "timing": 10984, + "##cted": 10985, + "shells": 10986, + "eased": 10987, + "geometry": 10988, + "louder": 10989, + "exciting": 10990, + "slovakia": 10991, + "##sion": 10992, + "##iz": 10993, + "##lot": 10994, + "savings": 10995, + "prairie": 10996, + "##ques": 10997, + "marching": 10998, + "rafael": 10999, + "tonnes": 11000, + "##lled": 11001, + "curtain": 11002, + "preceding": 11003, + "shy": 11004, + "heal": 11005, + "greene": 11006, + "worthy": 11007, + "##pot": 11008, + "detachment": 11009, + "bury": 11010, + "sherman": 11011, + "##eck": 11012, + "reinforced": 11013, + "seeks": 11014, + "bottles": 11015, + "contracted": 11016, + "duchess": 11017, + "outfit": 11018, + "walsh": 11019, + "##sc": 11020, + "mickey": 11021, + "##ase": 11022, + "geoffrey": 11023, + "archer": 11024, + "squeeze": 11025, + "dawson": 11026, + "eliminate": 11027, + "invention": 11028, + "##enberg": 11029, + "neal": 11030, + "##eth": 11031, + "stance": 11032, + "dealer": 11033, + "coral": 11034, + "maple": 11035, + "retire": 11036, + "polo": 11037, + "simplified": 11038, + "##ht": 11039, + "1833": 11040, + "hid": 11041, + "watts": 11042, + "backwards": 11043, + "jules": 11044, + "##oke": 11045, + "genesis": 11046, + "mt": 11047, + "frames": 11048, + "rebounds": 11049, + "burma": 11050, + "woodland": 11051, + "moist": 11052, + "santos": 11053, + "whispers": 11054, + "drained": 11055, + "subspecies": 11056, + "##aa": 11057, + "streaming": 11058, + "ulster": 11059, + "burnt": 11060, + "correspondence": 11061, + "maternal": 11062, + "gerard": 11063, + "denis": 11064, + "stealing": 11065, + "##load": 11066, + "genius": 11067, + "duchy": 11068, + "##oria": 11069, + "inaugurated": 11070, + "momentum": 11071, + "suits": 11072, + "placement": 11073, + "sovereign": 11074, + "clause": 11075, + "thames": 11076, + "##hara": 11077, + "confederation": 11078, + "reservation": 11079, + "sketch": 11080, + "yankees": 11081, + "lets": 11082, + "rotten": 11083, + "charm": 11084, + "hal": 11085, + "verses": 11086, + "ultra": 11087, + "commercially": 11088, + "dot": 11089, + "salon": 11090, + "citation": 11091, + "adopt": 11092, + "winnipeg": 11093, + "mist": 11094, + "allocated": 11095, + "cairo": 11096, + "##boy": 11097, + "jenkins": 11098, + "interference": 11099, + "objectives": 11100, + "##wind": 11101, + "1820": 11102, + "portfolio": 11103, + "armoured": 11104, + "sectors": 11105, + "##eh": 11106, + "initiatives": 11107, + "##world": 11108, + "integrity": 11109, + "exercises": 11110, + "robe": 11111, + "tap": 11112, + "ab": 11113, + "gazed": 11114, + "##tones": 11115, + "distracted": 11116, + "rulers": 11117, + "111": 11118, + "favorable": 11119, + "jerome": 11120, + "tended": 11121, + "cart": 11122, + "factories": 11123, + "##eri": 11124, + "diplomat": 11125, + "valued": 11126, + "gravel": 11127, + "charitable": 11128, + "##try": 11129, + "calvin": 11130, + "exploring": 11131, + "chang": 11132, + "shepherd": 11133, + "terrace": 11134, + "pdf": 11135, + "pupil": 11136, + "##ural": 11137, + "reflects": 11138, + "ups": 11139, + "##rch": 11140, + "governors": 11141, + "shelf": 11142, + "depths": 11143, + "##nberg": 11144, + "trailed": 11145, + "crest": 11146, + "tackle": 11147, + "##nian": 11148, + "##ats": 11149, + "hatred": 11150, + "##kai": 11151, + "clare": 11152, + "makers": 11153, + "ethiopia": 11154, + "longtime": 11155, + "detected": 11156, + "embedded": 11157, + "lacking": 11158, + "slapped": 11159, + "rely": 11160, + "thomson": 11161, + "anticipation": 11162, + "iso": 11163, + "morton": 11164, + "successive": 11165, + "agnes": 11166, + "screenwriter": 11167, + "straightened": 11168, + "philippe": 11169, + "playwright": 11170, + "haunted": 11171, + "licence": 11172, + "iris": 11173, + "intentions": 11174, + "sutton": 11175, + "112": 11176, + "logical": 11177, + "correctly": 11178, + "##weight": 11179, + "branded": 11180, + "licked": 11181, + "tipped": 11182, + "silva": 11183, + "ricky": 11184, + "narrator": 11185, + "requests": 11186, + "##ents": 11187, + "greeted": 11188, + "supernatural": 11189, + "cow": 11190, + "##wald": 11191, + "lung": 11192, + "refusing": 11193, + "employer": 11194, + "strait": 11195, + "gaelic": 11196, + "liner": 11197, + "##piece": 11198, + "zoe": 11199, + "sabha": 11200, + "##mba": 11201, + "driveway": 11202, + "harvest": 11203, + "prints": 11204, + "bates": 11205, + "reluctantly": 11206, + "threshold": 11207, + "algebra": 11208, + "ira": 11209, + "wherever": 11210, + "coupled": 11211, + "240": 11212, + "assumption": 11213, + "picks": 11214, + "##air": 11215, + "designers": 11216, + "raids": 11217, + "gentlemen": 11218, + "##ean": 11219, + "roller": 11220, + "blowing": 11221, + "leipzig": 11222, + "locks": 11223, + "screw": 11224, + "dressing": 11225, + "strand": 11226, + "##lings": 11227, + "scar": 11228, + "dwarf": 11229, + "depicts": 11230, + "##nu": 11231, + "nods": 11232, + "##mine": 11233, + "differ": 11234, + "boris": 11235, + "##eur": 11236, + "yuan": 11237, + "flip": 11238, + "##gie": 11239, + "mob": 11240, + "invested": 11241, + "questioning": 11242, + "applying": 11243, + "##ture": 11244, + "shout": 11245, + "##sel": 11246, + "gameplay": 11247, + "blamed": 11248, + "illustrations": 11249, + "bothered": 11250, + "weakness": 11251, + "rehabilitation": 11252, + "##of": 11253, + "##zes": 11254, + "envelope": 11255, + "rumors": 11256, + "miners": 11257, + "leicester": 11258, + "subtle": 11259, + "kerry": 11260, + "##ico": 11261, + "ferguson": 11262, + "##fu": 11263, + "premiership": 11264, + "ne": 11265, + "##cat": 11266, + "bengali": 11267, + "prof": 11268, + "catches": 11269, + "remnants": 11270, + "dana": 11271, + "##rily": 11272, + "shouting": 11273, + "presidents": 11274, + "baltic": 11275, + "ought": 11276, + "ghosts": 11277, + "dances": 11278, + "sailors": 11279, + "shirley": 11280, + "fancy": 11281, + "dominic": 11282, + "##bie": 11283, + "madonna": 11284, + "##rick": 11285, + "bark": 11286, + "buttons": 11287, + "gymnasium": 11288, + "ashes": 11289, + "liver": 11290, + "toby": 11291, + "oath": 11292, + "providence": 11293, + "doyle": 11294, + "evangelical": 11295, + "nixon": 11296, + "cement": 11297, + "carnegie": 11298, + "embarked": 11299, + "hatch": 11300, + "surroundings": 11301, + "guarantee": 11302, + "needing": 11303, + "pirate": 11304, + "essence": 11305, + "##bee": 11306, + "filter": 11307, + "crane": 11308, + "hammond": 11309, + "projected": 11310, + "immune": 11311, + "percy": 11312, + "twelfth": 11313, + "##ult": 11314, + "regent": 11315, + "doctoral": 11316, + "damon": 11317, + "mikhail": 11318, + "##ichi": 11319, + "lu": 11320, + "critically": 11321, + "elect": 11322, + "realised": 11323, + "abortion": 11324, + "acute": 11325, + "screening": 11326, + "mythology": 11327, + "steadily": 11328, + "##fc": 11329, + "frown": 11330, + "nottingham": 11331, + "kirk": 11332, + "wa": 11333, + "minneapolis": 11334, + "##rra": 11335, + "module": 11336, + "algeria": 11337, + "mc": 11338, + "nautical": 11339, + "encounters": 11340, + "surprising": 11341, + "statues": 11342, + "availability": 11343, + "shirts": 11344, + "pie": 11345, + "alma": 11346, + "brows": 11347, + "munster": 11348, + "mack": 11349, + "soup": 11350, + "crater": 11351, + "tornado": 11352, + "sanskrit": 11353, + "cedar": 11354, + "explosive": 11355, + "bordered": 11356, + "dixon": 11357, + "planets": 11358, + "stamp": 11359, + "exam": 11360, + "happily": 11361, + "##bble": 11362, + "carriers": 11363, + "kidnapped": 11364, + "##vis": 11365, + "accommodation": 11366, + "emigrated": 11367, + "##met": 11368, + "knockout": 11369, + "correspondent": 11370, + "violation": 11371, + "profits": 11372, + "peaks": 11373, + "lang": 11374, + "specimen": 11375, + "agenda": 11376, + "ancestry": 11377, + "pottery": 11378, + "spelling": 11379, + "equations": 11380, + "obtaining": 11381, + "ki": 11382, + "linking": 11383, + "1825": 11384, + "debris": 11385, + "asylum": 11386, + "##20": 11387, + "buddhism": 11388, + "teddy": 11389, + "##ants": 11390, + "gazette": 11391, + "##nger": 11392, + "##sse": 11393, + "dental": 11394, + "eligibility": 11395, + "utc": 11396, + "fathers": 11397, + "averaged": 11398, + "zimbabwe": 11399, + "francesco": 11400, + "coloured": 11401, + "hissed": 11402, + "translator": 11403, + "lynch": 11404, + "mandate": 11405, + "humanities": 11406, + "mackenzie": 11407, + "uniforms": 11408, + "lin": 11409, + "##iana": 11410, + "##gio": 11411, + "asset": 11412, + "mhz": 11413, + "fitting": 11414, + "samantha": 11415, + "genera": 11416, + "wei": 11417, + "rim": 11418, + "beloved": 11419, + "shark": 11420, + "riot": 11421, + "entities": 11422, + "expressions": 11423, + "indo": 11424, + "carmen": 11425, + "slipping": 11426, + "owing": 11427, + "abbot": 11428, + "neighbor": 11429, + "sidney": 11430, + "##av": 11431, + "rats": 11432, + "recommendations": 11433, + "encouraging": 11434, + "squadrons": 11435, + "anticipated": 11436, + "commanders": 11437, + "conquered": 11438, + "##oto": 11439, + "donations": 11440, + "diagnosed": 11441, + "##mond": 11442, + "divide": 11443, + "##iva": 11444, + "guessed": 11445, + "decoration": 11446, + "vernon": 11447, + "auditorium": 11448, + "revelation": 11449, + "conversations": 11450, + "##kers": 11451, + "##power": 11452, + "herzegovina": 11453, + "dash": 11454, + "alike": 11455, + "protested": 11456, + "lateral": 11457, + "herman": 11458, + "accredited": 11459, + "mg": 11460, + "##gent": 11461, + "freeman": 11462, + "mel": 11463, + "fiji": 11464, + "crow": 11465, + "crimson": 11466, + "##rine": 11467, + "livestock": 11468, + "##pped": 11469, + "humanitarian": 11470, + "bored": 11471, + "oz": 11472, + "whip": 11473, + "##lene": 11474, + "##ali": 11475, + "legitimate": 11476, + "alter": 11477, + "grinning": 11478, + "spelled": 11479, + "anxious": 11480, + "oriental": 11481, + "wesley": 11482, + "##nin": 11483, + "##hole": 11484, + "carnival": 11485, + "controller": 11486, + "detect": 11487, + "##ssa": 11488, + "bowed": 11489, + "educator": 11490, + "kosovo": 11491, + "macedonia": 11492, + "##sin": 11493, + "occupy": 11494, + "mastering": 11495, + "stephanie": 11496, + "janeiro": 11497, + "para": 11498, + "unaware": 11499, + "nurses": 11500, + "noon": 11501, + "135": 11502, + "cam": 11503, + "hopefully": 11504, + "ranger": 11505, + "combine": 11506, + "sociology": 11507, + "polar": 11508, + "rica": 11509, + "##eer": 11510, + "neill": 11511, + "##sman": 11512, + "holocaust": 11513, + "##ip": 11514, + "doubled": 11515, + "lust": 11516, + "1828": 11517, + "109": 11518, + "decent": 11519, + "cooling": 11520, + "unveiled": 11521, + "##card": 11522, + "1829": 11523, + "nsw": 11524, + "homer": 11525, + "chapman": 11526, + "meyer": 11527, + "##gin": 11528, + "dive": 11529, + "mae": 11530, + "reagan": 11531, + "expertise": 11532, + "##gled": 11533, + "darwin": 11534, + "brooke": 11535, + "sided": 11536, + "prosecution": 11537, + "investigating": 11538, + "comprised": 11539, + "petroleum": 11540, + "genres": 11541, + "reluctant": 11542, + "differently": 11543, + "trilogy": 11544, + "johns": 11545, + "vegetables": 11546, + "corpse": 11547, + "highlighted": 11548, + "lounge": 11549, + "pension": 11550, + "unsuccessfully": 11551, + "elegant": 11552, + "aided": 11553, + "ivory": 11554, + "beatles": 11555, + "amelia": 11556, + "cain": 11557, + "dubai": 11558, + "sunny": 11559, + "immigrant": 11560, + "babe": 11561, + "click": 11562, + "##nder": 11563, + "underwater": 11564, + "pepper": 11565, + "combining": 11566, + "mumbled": 11567, + "atlas": 11568, + "horns": 11569, + "accessed": 11570, + "ballad": 11571, + "physicians": 11572, + "homeless": 11573, + "gestured": 11574, + "rpm": 11575, + "freak": 11576, + "louisville": 11577, + "corporations": 11578, + "patriots": 11579, + "prizes": 11580, + "rational": 11581, + "warn": 11582, + "modes": 11583, + "decorative": 11584, + "overnight": 11585, + "din": 11586, + "troubled": 11587, + "phantom": 11588, + "##ort": 11589, + "monarch": 11590, + "sheer": 11591, + "##dorf": 11592, + "generals": 11593, + "guidelines": 11594, + "organs": 11595, + "addresses": 11596, + "##zon": 11597, + "enhance": 11598, + "curling": 11599, + "parishes": 11600, + "cord": 11601, + "##kie": 11602, + "linux": 11603, + "caesar": 11604, + "deutsche": 11605, + "bavaria": 11606, + "##bia": 11607, + "coleman": 11608, + "cyclone": 11609, + "##eria": 11610, + "bacon": 11611, + "petty": 11612, + "##yama": 11613, + "##old": 11614, + "hampton": 11615, + "diagnosis": 11616, + "1824": 11617, + "throws": 11618, + "complexity": 11619, + "rita": 11620, + "disputed": 11621, + "##₃": 11622, + "pablo": 11623, + "##sch": 11624, + "marketed": 11625, + "trafficking": 11626, + "##ulus": 11627, + "examine": 11628, + "plague": 11629, + "formats": 11630, + "##oh": 11631, + "vault": 11632, + "faithful": 11633, + "##bourne": 11634, + "webster": 11635, + "##ox": 11636, + "highlights": 11637, + "##ient": 11638, + "##ann": 11639, + "phones": 11640, + "vacuum": 11641, + "sandwich": 11642, + "modeling": 11643, + "##gated": 11644, + "bolivia": 11645, + "clergy": 11646, + "qualities": 11647, + "isabel": 11648, + "##nas": 11649, + "##ars": 11650, + "wears": 11651, + "screams": 11652, + "reunited": 11653, + "annoyed": 11654, + "bra": 11655, + "##ancy": 11656, + "##rate": 11657, + "differential": 11658, + "transmitter": 11659, + "tattoo": 11660, + "container": 11661, + "poker": 11662, + "##och": 11663, + "excessive": 11664, + "resides": 11665, + "cowboys": 11666, + "##tum": 11667, + "augustus": 11668, + "trash": 11669, + "providers": 11670, + "statute": 11671, + "retreated": 11672, + "balcony": 11673, + "reversed": 11674, + "void": 11675, + "storey": 11676, + "preceded": 11677, + "masses": 11678, + "leap": 11679, + "laughs": 11680, + "neighborhoods": 11681, + "wards": 11682, + "schemes": 11683, + "falcon": 11684, + "santo": 11685, + "battlefield": 11686, + "pad": 11687, + "ronnie": 11688, + "thread": 11689, + "lesbian": 11690, + "venus": 11691, + "##dian": 11692, + "beg": 11693, + "sandstone": 11694, + "daylight": 11695, + "punched": 11696, + "gwen": 11697, + "analog": 11698, + "stroked": 11699, + "wwe": 11700, + "acceptable": 11701, + "measurements": 11702, + "dec": 11703, + "toxic": 11704, + "##kel": 11705, + "adequate": 11706, + "surgical": 11707, + "economist": 11708, + "parameters": 11709, + "varsity": 11710, + "##sberg": 11711, + "quantity": 11712, + "ella": 11713, + "##chy": 11714, + "##rton": 11715, + "countess": 11716, + "generating": 11717, + "precision": 11718, + "diamonds": 11719, + "expressway": 11720, + "ga": 11721, + "##ı": 11722, + "1821": 11723, + "uruguay": 11724, + "talents": 11725, + "galleries": 11726, + "expenses": 11727, + "scanned": 11728, + "colleague": 11729, + "outlets": 11730, + "ryder": 11731, + "lucien": 11732, + "##ila": 11733, + "paramount": 11734, + "##bon": 11735, + "syracuse": 11736, + "dim": 11737, + "fangs": 11738, + "gown": 11739, + "sweep": 11740, + "##sie": 11741, + "toyota": 11742, + "missionaries": 11743, + "websites": 11744, + "##nsis": 11745, + "sentences": 11746, + "adviser": 11747, + "val": 11748, + "trademark": 11749, + "spells": 11750, + "##plane": 11751, + "patience": 11752, + "starter": 11753, + "slim": 11754, + "##borg": 11755, + "toe": 11756, + "incredibly": 11757, + "shoots": 11758, + "elliot": 11759, + "nobility": 11760, + "##wyn": 11761, + "cowboy": 11762, + "endorsed": 11763, + "gardner": 11764, + "tendency": 11765, + "persuaded": 11766, + "organisms": 11767, + "emissions": 11768, + "kazakhstan": 11769, + "amused": 11770, + "boring": 11771, + "chips": 11772, + "themed": 11773, + "##hand": 11774, + "llc": 11775, + "constantinople": 11776, + "chasing": 11777, + "systematic": 11778, + "guatemala": 11779, + "borrowed": 11780, + "erin": 11781, + "carey": 11782, + "##hard": 11783, + "highlands": 11784, + "struggles": 11785, + "1810": 11786, + "##ifying": 11787, + "##ced": 11788, + "wong": 11789, + "exceptions": 11790, + "develops": 11791, + "enlarged": 11792, + "kindergarten": 11793, + "castro": 11794, + "##ern": 11795, + "##rina": 11796, + "leigh": 11797, + "zombie": 11798, + "juvenile": 11799, + "##most": 11800, + "consul": 11801, + "##nar": 11802, + "sailor": 11803, + "hyde": 11804, + "clarence": 11805, + "intensive": 11806, + "pinned": 11807, + "nasty": 11808, + "useless": 11809, + "jung": 11810, + "clayton": 11811, + "stuffed": 11812, + "exceptional": 11813, + "ix": 11814, + "apostolic": 11815, + "230": 11816, + "transactions": 11817, + "##dge": 11818, + "exempt": 11819, + "swinging": 11820, + "cove": 11821, + "religions": 11822, + "##ash": 11823, + "shields": 11824, + "dairy": 11825, + "bypass": 11826, + "190": 11827, + "pursuing": 11828, + "bug": 11829, + "joyce": 11830, + "bombay": 11831, + "chassis": 11832, + "southampton": 11833, + "chat": 11834, + "interact": 11835, + "redesignated": 11836, + "##pen": 11837, + "nascar": 11838, + "pray": 11839, + "salmon": 11840, + "rigid": 11841, + "regained": 11842, + "malaysian": 11843, + "grim": 11844, + "publicity": 11845, + "constituted": 11846, + "capturing": 11847, + "toilet": 11848, + "delegate": 11849, + "purely": 11850, + "tray": 11851, + "drift": 11852, + "loosely": 11853, + "striker": 11854, + "weakened": 11855, + "trinidad": 11856, + "mitch": 11857, + "itv": 11858, + "defines": 11859, + "transmitted": 11860, + "ming": 11861, + "scarlet": 11862, + "nodding": 11863, + "fitzgerald": 11864, + "fu": 11865, + "narrowly": 11866, + "sp": 11867, + "tooth": 11868, + "standings": 11869, + "virtue": 11870, + "##₁": 11871, + "##wara": 11872, + "##cting": 11873, + "chateau": 11874, + "gloves": 11875, + "lid": 11876, + "##nel": 11877, + "hurting": 11878, + "conservatory": 11879, + "##pel": 11880, + "sinclair": 11881, + "reopened": 11882, + "sympathy": 11883, + "nigerian": 11884, + "strode": 11885, + "advocated": 11886, + "optional": 11887, + "chronic": 11888, + "discharge": 11889, + "##rc": 11890, + "suck": 11891, + "compatible": 11892, + "laurel": 11893, + "stella": 11894, + "shi": 11895, + "fails": 11896, + "wage": 11897, + "dodge": 11898, + "128": 11899, + "informal": 11900, + "sorts": 11901, + "levi": 11902, + "buddha": 11903, + "villagers": 11904, + "##aka": 11905, + "chronicles": 11906, + "heavier": 11907, + "summoned": 11908, + "gateway": 11909, + "3000": 11910, + "eleventh": 11911, + "jewelry": 11912, + "translations": 11913, + "accordingly": 11914, + "seas": 11915, + "##ency": 11916, + "fiber": 11917, + "pyramid": 11918, + "cubic": 11919, + "dragging": 11920, + "##ista": 11921, + "caring": 11922, + "##ops": 11923, + "android": 11924, + "contacted": 11925, + "lunar": 11926, + "##dt": 11927, + "kai": 11928, + "lisbon": 11929, + "patted": 11930, + "1826": 11931, + "sacramento": 11932, + "theft": 11933, + "madagascar": 11934, + "subtropical": 11935, + "disputes": 11936, + "ta": 11937, + "holidays": 11938, + "piper": 11939, + "willow": 11940, + "mare": 11941, + "cane": 11942, + "itunes": 11943, + "newfoundland": 11944, + "benny": 11945, + "companions": 11946, + "dong": 11947, + "raj": 11948, + "observe": 11949, + "roar": 11950, + "charming": 11951, + "plaque": 11952, + "tibetan": 11953, + "fossils": 11954, + "enacted": 11955, + "manning": 11956, + "bubble": 11957, + "tina": 11958, + "tanzania": 11959, + "##eda": 11960, + "##hir": 11961, + "funk": 11962, + "swamp": 11963, + "deputies": 11964, + "cloak": 11965, + "ufc": 11966, + "scenario": 11967, + "par": 11968, + "scratch": 11969, + "metals": 11970, + "anthem": 11971, + "guru": 11972, + "engaging": 11973, + "specially": 11974, + "##boat": 11975, + "dialects": 11976, + "nineteen": 11977, + "cecil": 11978, + "duet": 11979, + "disability": 11980, + "messenger": 11981, + "unofficial": 11982, + "##lies": 11983, + "defunct": 11984, + "eds": 11985, + "moonlight": 11986, + "drainage": 11987, + "surname": 11988, + "puzzle": 11989, + "honda": 11990, + "switching": 11991, + "conservatives": 11992, + "mammals": 11993, + "knox": 11994, + "broadcaster": 11995, + "sidewalk": 11996, + "cope": 11997, + "##ried": 11998, + "benson": 11999, + "princes": 12000, + "peterson": 12001, + "##sal": 12002, + "bedford": 12003, + "sharks": 12004, + "eli": 12005, + "wreck": 12006, + "alberto": 12007, + "gasp": 12008, + "archaeology": 12009, + "lgbt": 12010, + "teaches": 12011, + "securities": 12012, + "madness": 12013, + "compromise": 12014, + "waving": 12015, + "coordination": 12016, + "davidson": 12017, + "visions": 12018, + "leased": 12019, + "possibilities": 12020, + "eighty": 12021, + "jun": 12022, + "fernandez": 12023, + "enthusiasm": 12024, + "assassin": 12025, + "sponsorship": 12026, + "reviewer": 12027, + "kingdoms": 12028, + "estonian": 12029, + "laboratories": 12030, + "##fy": 12031, + "##nal": 12032, + "applies": 12033, + "verb": 12034, + "celebrations": 12035, + "##zzo": 12036, + "rowing": 12037, + "lightweight": 12038, + "sadness": 12039, + "submit": 12040, + "mvp": 12041, + "balanced": 12042, + "dude": 12043, + "##vas": 12044, + "explicitly": 12045, + "metric": 12046, + "magnificent": 12047, + "mound": 12048, + "brett": 12049, + "mohammad": 12050, + "mistakes": 12051, + "irregular": 12052, + "##hing": 12053, + "##ass": 12054, + "sanders": 12055, + "betrayed": 12056, + "shipped": 12057, + "surge": 12058, + "##enburg": 12059, + "reporters": 12060, + "termed": 12061, + "georg": 12062, + "pity": 12063, + "verbal": 12064, + "bulls": 12065, + "abbreviated": 12066, + "enabling": 12067, + "appealed": 12068, + "##are": 12069, + "##atic": 12070, + "sicily": 12071, + "sting": 12072, + "heel": 12073, + "sweetheart": 12074, + "bart": 12075, + "spacecraft": 12076, + "brutal": 12077, + "monarchy": 12078, + "##tter": 12079, + "aberdeen": 12080, + "cameo": 12081, + "diane": 12082, + "##ub": 12083, + "survivor": 12084, + "clyde": 12085, + "##aries": 12086, + "complaint": 12087, + "##makers": 12088, + "clarinet": 12089, + "delicious": 12090, + "chilean": 12091, + "karnataka": 12092, + "coordinates": 12093, + "1818": 12094, + "panties": 12095, + "##rst": 12096, + "pretending": 12097, + "ar": 12098, + "dramatically": 12099, + "kiev": 12100, + "bella": 12101, + "tends": 12102, + "distances": 12103, + "113": 12104, + "catalog": 12105, + "launching": 12106, + "instances": 12107, + "telecommunications": 12108, + "portable": 12109, + "lindsay": 12110, + "vatican": 12111, + "##eim": 12112, + "angles": 12113, + "aliens": 12114, + "marker": 12115, + "stint": 12116, + "screens": 12117, + "bolton": 12118, + "##rne": 12119, + "judy": 12120, + "wool": 12121, + "benedict": 12122, + "plasma": 12123, + "europa": 12124, + "spark": 12125, + "imaging": 12126, + "filmmaker": 12127, + "swiftly": 12128, + "##een": 12129, + "contributor": 12130, + "##nor": 12131, + "opted": 12132, + "stamps": 12133, + "apologize": 12134, + "financing": 12135, + "butter": 12136, + "gideon": 12137, + "sophisticated": 12138, + "alignment": 12139, + "avery": 12140, + "chemicals": 12141, + "yearly": 12142, + "speculation": 12143, + "prominence": 12144, + "professionally": 12145, + "##ils": 12146, + "immortal": 12147, + "institutional": 12148, + "inception": 12149, + "wrists": 12150, + "identifying": 12151, + "tribunal": 12152, + "derives": 12153, + "gains": 12154, + "##wo": 12155, + "papal": 12156, + "preference": 12157, + "linguistic": 12158, + "vince": 12159, + "operative": 12160, + "brewery": 12161, + "##ont": 12162, + "unemployment": 12163, + "boyd": 12164, + "##ured": 12165, + "##outs": 12166, + "albeit": 12167, + "prophet": 12168, + "1813": 12169, + "bi": 12170, + "##rr": 12171, + "##face": 12172, + "##rad": 12173, + "quarterly": 12174, + "asteroid": 12175, + "cleaned": 12176, + "radius": 12177, + "temper": 12178, + "##llen": 12179, + "telugu": 12180, + "jerk": 12181, + "viscount": 12182, + "menu": 12183, + "##ote": 12184, + "glimpse": 12185, + "##aya": 12186, + "yacht": 12187, + "hawaiian": 12188, + "baden": 12189, + "##rl": 12190, + "laptop": 12191, + "readily": 12192, + "##gu": 12193, + "monetary": 12194, + "offshore": 12195, + "scots": 12196, + "watches": 12197, + "##yang": 12198, + "##arian": 12199, + "upgrade": 12200, + "needle": 12201, + "xbox": 12202, + "lea": 12203, + "encyclopedia": 12204, + "flank": 12205, + "fingertips": 12206, + "##pus": 12207, + "delight": 12208, + "teachings": 12209, + "confirm": 12210, + "roth": 12211, + "beaches": 12212, + "midway": 12213, + "winters": 12214, + "##iah": 12215, + "teasing": 12216, + "daytime": 12217, + "beverly": 12218, + "gambling": 12219, + "bonnie": 12220, + "##backs": 12221, + "regulated": 12222, + "clement": 12223, + "hermann": 12224, + "tricks": 12225, + "knot": 12226, + "##shing": 12227, + "##uring": 12228, + "##vre": 12229, + "detached": 12230, + "ecological": 12231, + "owed": 12232, + "specialty": 12233, + "byron": 12234, + "inventor": 12235, + "bats": 12236, + "stays": 12237, + "screened": 12238, + "unesco": 12239, + "midland": 12240, + "trim": 12241, + "affection": 12242, + "##ander": 12243, + "##rry": 12244, + "jess": 12245, + "thoroughly": 12246, + "feedback": 12247, + "##uma": 12248, + "chennai": 12249, + "strained": 12250, + "heartbeat": 12251, + "wrapping": 12252, + "overtime": 12253, + "pleaded": 12254, + "##sworth": 12255, + "mon": 12256, + "leisure": 12257, + "oclc": 12258, + "##tate": 12259, + "##ele": 12260, + "feathers": 12261, + "angelo": 12262, + "thirds": 12263, + "nuts": 12264, + "surveys": 12265, + "clever": 12266, + "gill": 12267, + "commentator": 12268, + "##dos": 12269, + "darren": 12270, + "rides": 12271, + "gibraltar": 12272, + "##nc": 12273, + "##mu": 12274, + "dissolution": 12275, + "dedication": 12276, + "shin": 12277, + "meals": 12278, + "saddle": 12279, + "elvis": 12280, + "reds": 12281, + "chaired": 12282, + "taller": 12283, + "appreciation": 12284, + "functioning": 12285, + "niece": 12286, + "favored": 12287, + "advocacy": 12288, + "robbie": 12289, + "criminals": 12290, + "suffolk": 12291, + "yugoslav": 12292, + "passport": 12293, + "constable": 12294, + "congressman": 12295, + "hastings": 12296, + "vera": 12297, + "##rov": 12298, + "consecrated": 12299, + "sparks": 12300, + "ecclesiastical": 12301, + "confined": 12302, + "##ovich": 12303, + "muller": 12304, + "floyd": 12305, + "nora": 12306, + "1822": 12307, + "paved": 12308, + "1827": 12309, + "cumberland": 12310, + "ned": 12311, + "saga": 12312, + "spiral": 12313, + "##flow": 12314, + "appreciated": 12315, + "yi": 12316, + "collaborative": 12317, + "treating": 12318, + "similarities": 12319, + "feminine": 12320, + "finishes": 12321, + "##ib": 12322, + "jade": 12323, + "import": 12324, + "##nse": 12325, + "##hot": 12326, + "champagne": 12327, + "mice": 12328, + "securing": 12329, + "celebrities": 12330, + "helsinki": 12331, + "attributes": 12332, + "##gos": 12333, + "cousins": 12334, + "phases": 12335, + "ache": 12336, + "lucia": 12337, + "gandhi": 12338, + "submission": 12339, + "vicar": 12340, + "spear": 12341, + "shine": 12342, + "tasmania": 12343, + "biting": 12344, + "detention": 12345, + "constitute": 12346, + "tighter": 12347, + "seasonal": 12348, + "##gus": 12349, + "terrestrial": 12350, + "matthews": 12351, + "##oka": 12352, + "effectiveness": 12353, + "parody": 12354, + "philharmonic": 12355, + "##onic": 12356, + "1816": 12357, + "strangers": 12358, + "encoded": 12359, + "consortium": 12360, + "guaranteed": 12361, + "regards": 12362, + "shifts": 12363, + "tortured": 12364, + "collision": 12365, + "supervisor": 12366, + "inform": 12367, + "broader": 12368, + "insight": 12369, + "theaters": 12370, + "armour": 12371, + "emeritus": 12372, + "blink": 12373, + "incorporates": 12374, + "mapping": 12375, + "##50": 12376, + "##ein": 12377, + "handball": 12378, + "flexible": 12379, + "##nta": 12380, + "substantially": 12381, + "generous": 12382, + "thief": 12383, + "##own": 12384, + "carr": 12385, + "loses": 12386, + "1793": 12387, + "prose": 12388, + "ucla": 12389, + "romeo": 12390, + "generic": 12391, + "metallic": 12392, + "realization": 12393, + "damages": 12394, + "mk": 12395, + "commissioners": 12396, + "zach": 12397, + "default": 12398, + "##ther": 12399, + "helicopters": 12400, + "lengthy": 12401, + "stems": 12402, + "spa": 12403, + "partnered": 12404, + "spectators": 12405, + "rogue": 12406, + "indication": 12407, + "penalties": 12408, + "teresa": 12409, + "1801": 12410, + "sen": 12411, + "##tric": 12412, + "dalton": 12413, + "##wich": 12414, + "irving": 12415, + "photographic": 12416, + "##vey": 12417, + "dell": 12418, + "deaf": 12419, + "peters": 12420, + "excluded": 12421, + "unsure": 12422, + "##vable": 12423, + "patterson": 12424, + "crawled": 12425, + "##zio": 12426, + "resided": 12427, + "whipped": 12428, + "latvia": 12429, + "slower": 12430, + "ecole": 12431, + "pipes": 12432, + "employers": 12433, + "maharashtra": 12434, + "comparable": 12435, + "va": 12436, + "textile": 12437, + "pageant": 12438, + "##gel": 12439, + "alphabet": 12440, + "binary": 12441, + "irrigation": 12442, + "chartered": 12443, + "choked": 12444, + "antoine": 12445, + "offs": 12446, + "waking": 12447, + "supplement": 12448, + "##wen": 12449, + "quantities": 12450, + "demolition": 12451, + "regain": 12452, + "locate": 12453, + "urdu": 12454, + "folks": 12455, + "alt": 12456, + "114": 12457, + "##mc": 12458, + "scary": 12459, + "andreas": 12460, + "whites": 12461, + "##ava": 12462, + "classrooms": 12463, + "mw": 12464, + "aesthetic": 12465, + "publishes": 12466, + "valleys": 12467, + "guides": 12468, + "cubs": 12469, + "johannes": 12470, + "bryant": 12471, + "conventions": 12472, + "affecting": 12473, + "##itt": 12474, + "drain": 12475, + "awesome": 12476, + "isolation": 12477, + "prosecutor": 12478, + "ambitious": 12479, + "apology": 12480, + "captive": 12481, + "downs": 12482, + "atmospheric": 12483, + "lorenzo": 12484, + "aisle": 12485, + "beef": 12486, + "foul": 12487, + "##onia": 12488, + "kidding": 12489, + "composite": 12490, + "disturbed": 12491, + "illusion": 12492, + "natives": 12493, + "##ffer": 12494, + "emi": 12495, + "rockets": 12496, + "riverside": 12497, + "wartime": 12498, + "painters": 12499, + "adolf": 12500, + "melted": 12501, + "##ail": 12502, + "uncertainty": 12503, + "simulation": 12504, + "hawks": 12505, + "progressed": 12506, + "meantime": 12507, + "builder": 12508, + "spray": 12509, + "breach": 12510, + "unhappy": 12511, + "regina": 12512, + "russians": 12513, + "##urg": 12514, + "determining": 12515, + "##tation": 12516, + "tram": 12517, + "1806": 12518, + "##quin": 12519, + "aging": 12520, + "##12": 12521, + "1823": 12522, + "garion": 12523, + "rented": 12524, + "mister": 12525, + "diaz": 12526, + "terminated": 12527, + "clip": 12528, + "1817": 12529, + "depend": 12530, + "nervously": 12531, + "disco": 12532, + "owe": 12533, + "defenders": 12534, + "shiva": 12535, + "notorious": 12536, + "disbelief": 12537, + "shiny": 12538, + "worcester": 12539, + "##gation": 12540, + "##yr": 12541, + "trailing": 12542, + "undertook": 12543, + "islander": 12544, + "belarus": 12545, + "limitations": 12546, + "watershed": 12547, + "fuller": 12548, + "overlooking": 12549, + "utilized": 12550, + "raphael": 12551, + "1819": 12552, + "synthetic": 12553, + "breakdown": 12554, + "klein": 12555, + "##nate": 12556, + "moaned": 12557, + "memoir": 12558, + "lamb": 12559, + "practicing": 12560, + "##erly": 12561, + "cellular": 12562, + "arrows": 12563, + "exotic": 12564, + "##graphy": 12565, + "witches": 12566, + "117": 12567, + "charted": 12568, + "rey": 12569, + "hut": 12570, + "hierarchy": 12571, + "subdivision": 12572, + "freshwater": 12573, + "giuseppe": 12574, + "aloud": 12575, + "reyes": 12576, + "qatar": 12577, + "marty": 12578, + "sideways": 12579, + "utterly": 12580, + "sexually": 12581, + "jude": 12582, + "prayers": 12583, + "mccarthy": 12584, + "softball": 12585, + "blend": 12586, + "damien": 12587, + "##gging": 12588, + "##metric": 12589, + "wholly": 12590, + "erupted": 12591, + "lebanese": 12592, + "negro": 12593, + "revenues": 12594, + "tasted": 12595, + "comparative": 12596, + "teamed": 12597, + "transaction": 12598, + "labeled": 12599, + "maori": 12600, + "sovereignty": 12601, + "parkway": 12602, + "trauma": 12603, + "gran": 12604, + "malay": 12605, + "121": 12606, + "advancement": 12607, + "descendant": 12608, + "2020": 12609, + "buzz": 12610, + "salvation": 12611, + "inventory": 12612, + "symbolic": 12613, + "##making": 12614, + "antarctica": 12615, + "mps": 12616, + "##gas": 12617, + "##bro": 12618, + "mohammed": 12619, + "myanmar": 12620, + "holt": 12621, + "submarines": 12622, + "tones": 12623, + "##lman": 12624, + "locker": 12625, + "patriarch": 12626, + "bangkok": 12627, + "emerson": 12628, + "remarks": 12629, + "predators": 12630, + "kin": 12631, + "afghan": 12632, + "confession": 12633, + "norwich": 12634, + "rental": 12635, + "emerge": 12636, + "advantages": 12637, + "##zel": 12638, + "rca": 12639, + "##hold": 12640, + "shortened": 12641, + "storms": 12642, + "aidan": 12643, + "##matic": 12644, + "autonomy": 12645, + "compliance": 12646, + "##quet": 12647, + "dudley": 12648, + "atp": 12649, + "##osis": 12650, + "1803": 12651, + "motto": 12652, + "documentation": 12653, + "summary": 12654, + "professors": 12655, + "spectacular": 12656, + "christina": 12657, + "archdiocese": 12658, + "flashing": 12659, + "innocence": 12660, + "remake": 12661, + "##dell": 12662, + "psychic": 12663, + "reef": 12664, + "scare": 12665, + "employ": 12666, + "rs": 12667, + "sticks": 12668, + "meg": 12669, + "gus": 12670, + "leans": 12671, + "##ude": 12672, + "accompany": 12673, + "bergen": 12674, + "tomas": 12675, + "##iko": 12676, + "doom": 12677, + "wages": 12678, + "pools": 12679, + "##nch": 12680, + "##bes": 12681, + "breasts": 12682, + "scholarly": 12683, + "alison": 12684, + "outline": 12685, + "brittany": 12686, + "breakthrough": 12687, + "willis": 12688, + "realistic": 12689, + "##cut": 12690, + "##boro": 12691, + "competitor": 12692, + "##stan": 12693, + "pike": 12694, + "picnic": 12695, + "icon": 12696, + "designing": 12697, + "commercials": 12698, + "washing": 12699, + "villain": 12700, + "skiing": 12701, + "micro": 12702, + "costumes": 12703, + "auburn": 12704, + "halted": 12705, + "executives": 12706, + "##hat": 12707, + "logistics": 12708, + "cycles": 12709, + "vowel": 12710, + "applicable": 12711, + "barrett": 12712, + "exclaimed": 12713, + "eurovision": 12714, + "eternity": 12715, + "ramon": 12716, + "##umi": 12717, + "##lls": 12718, + "modifications": 12719, + "sweeping": 12720, + "disgust": 12721, + "##uck": 12722, + "torch": 12723, + "aviv": 12724, + "ensuring": 12725, + "rude": 12726, + "dusty": 12727, + "sonic": 12728, + "donovan": 12729, + "outskirts": 12730, + "cu": 12731, + "pathway": 12732, + "##band": 12733, + "##gun": 12734, + "##lines": 12735, + "disciplines": 12736, + "acids": 12737, + "cadet": 12738, + "paired": 12739, + "##40": 12740, + "sketches": 12741, + "##sive": 12742, + "marriages": 12743, + "##⁺": 12744, + "folding": 12745, + "peers": 12746, + "slovak": 12747, + "implies": 12748, + "admired": 12749, + "##beck": 12750, + "1880s": 12751, + "leopold": 12752, + "instinct": 12753, + "attained": 12754, + "weston": 12755, + "megan": 12756, + "horace": 12757, + "##ination": 12758, + "dorsal": 12759, + "ingredients": 12760, + "evolutionary": 12761, + "##its": 12762, + "complications": 12763, + "deity": 12764, + "lethal": 12765, + "brushing": 12766, + "levy": 12767, + "deserted": 12768, + "institutes": 12769, + "posthumously": 12770, + "delivering": 12771, + "telescope": 12772, + "coronation": 12773, + "motivated": 12774, + "rapids": 12775, + "luc": 12776, + "flicked": 12777, + "pays": 12778, + "volcano": 12779, + "tanner": 12780, + "weighed": 12781, + "##nica": 12782, + "crowds": 12783, + "frankie": 12784, + "gifted": 12785, + "addressing": 12786, + "granddaughter": 12787, + "winding": 12788, + "##rna": 12789, + "constantine": 12790, + "gomez": 12791, + "##front": 12792, + "landscapes": 12793, + "rudolf": 12794, + "anthropology": 12795, + "slate": 12796, + "werewolf": 12797, + "##lio": 12798, + "astronomy": 12799, + "circa": 12800, + "rouge": 12801, + "dreaming": 12802, + "sack": 12803, + "knelt": 12804, + "drowned": 12805, + "naomi": 12806, + "prolific": 12807, + "tracked": 12808, + "freezing": 12809, + "herb": 12810, + "##dium": 12811, + "agony": 12812, + "randall": 12813, + "twisting": 12814, + "wendy": 12815, + "deposit": 12816, + "touches": 12817, + "vein": 12818, + "wheeler": 12819, + "##bbled": 12820, + "##bor": 12821, + "batted": 12822, + "retaining": 12823, + "tire": 12824, + "presently": 12825, + "compare": 12826, + "specification": 12827, + "daemon": 12828, + "nigel": 12829, + "##grave": 12830, + "merry": 12831, + "recommendation": 12832, + "czechoslovakia": 12833, + "sandra": 12834, + "ng": 12835, + "roma": 12836, + "##sts": 12837, + "lambert": 12838, + "inheritance": 12839, + "sheikh": 12840, + "winchester": 12841, + "cries": 12842, + "examining": 12843, + "##yle": 12844, + "comeback": 12845, + "cuisine": 12846, + "nave": 12847, + "##iv": 12848, + "ko": 12849, + "retrieve": 12850, + "tomatoes": 12851, + "barker": 12852, + "polished": 12853, + "defining": 12854, + "irene": 12855, + "lantern": 12856, + "personalities": 12857, + "begging": 12858, + "tract": 12859, + "swore": 12860, + "1809": 12861, + "175": 12862, + "##gic": 12863, + "omaha": 12864, + "brotherhood": 12865, + "##rley": 12866, + "haiti": 12867, + "##ots": 12868, + "exeter": 12869, + "##ete": 12870, + "##zia": 12871, + "steele": 12872, + "dumb": 12873, + "pearson": 12874, + "210": 12875, + "surveyed": 12876, + "elisabeth": 12877, + "trends": 12878, + "##ef": 12879, + "fritz": 12880, + "##rf": 12881, + "premium": 12882, + "bugs": 12883, + "fraction": 12884, + "calmly": 12885, + "viking": 12886, + "##birds": 12887, + "tug": 12888, + "inserted": 12889, + "unusually": 12890, + "##ield": 12891, + "confronted": 12892, + "distress": 12893, + "crashing": 12894, + "brent": 12895, + "turks": 12896, + "resign": 12897, + "##olo": 12898, + "cambodia": 12899, + "gabe": 12900, + "sauce": 12901, + "##kal": 12902, + "evelyn": 12903, + "116": 12904, + "extant": 12905, + "clusters": 12906, + "quarry": 12907, + "teenagers": 12908, + "luna": 12909, + "##lers": 12910, + "##ister": 12911, + "affiliation": 12912, + "drill": 12913, + "##ashi": 12914, + "panthers": 12915, + "scenic": 12916, + "libya": 12917, + "anita": 12918, + "strengthen": 12919, + "inscriptions": 12920, + "##cated": 12921, + "lace": 12922, + "sued": 12923, + "judith": 12924, + "riots": 12925, + "##uted": 12926, + "mint": 12927, + "##eta": 12928, + "preparations": 12929, + "midst": 12930, + "dub": 12931, + "challenger": 12932, + "##vich": 12933, + "mock": 12934, + "cf": 12935, + "displaced": 12936, + "wicket": 12937, + "breaths": 12938, + "enables": 12939, + "schmidt": 12940, + "analyst": 12941, + "##lum": 12942, + "ag": 12943, + "highlight": 12944, + "automotive": 12945, + "axe": 12946, + "josef": 12947, + "newark": 12948, + "sufficiently": 12949, + "resembles": 12950, + "50th": 12951, + "##pal": 12952, + "flushed": 12953, + "mum": 12954, + "traits": 12955, + "##ante": 12956, + "commodore": 12957, + "incomplete": 12958, + "warming": 12959, + "titular": 12960, + "ceremonial": 12961, + "ethical": 12962, + "118": 12963, + "celebrating": 12964, + "eighteenth": 12965, + "cao": 12966, + "lima": 12967, + "medalist": 12968, + "mobility": 12969, + "strips": 12970, + "snakes": 12971, + "##city": 12972, + "miniature": 12973, + "zagreb": 12974, + "barton": 12975, + "escapes": 12976, + "umbrella": 12977, + "automated": 12978, + "doubted": 12979, + "differs": 12980, + "cooled": 12981, + "georgetown": 12982, + "dresden": 12983, + "cooked": 12984, + "fade": 12985, + "wyatt": 12986, + "rna": 12987, + "jacobs": 12988, + "carlton": 12989, + "abundant": 12990, + "stereo": 12991, + "boost": 12992, + "madras": 12993, + "inning": 12994, + "##hia": 12995, + "spur": 12996, + "ip": 12997, + "malayalam": 12998, + "begged": 12999, + "osaka": 13000, + "groan": 13001, + "escaping": 13002, + "charging": 13003, + "dose": 13004, + "vista": 13005, + "##aj": 13006, + "bud": 13007, + "papa": 13008, + "communists": 13009, + "advocates": 13010, + "edged": 13011, + "tri": 13012, + "##cent": 13013, + "resemble": 13014, + "peaking": 13015, + "necklace": 13016, + "fried": 13017, + "montenegro": 13018, + "saxony": 13019, + "goose": 13020, + "glances": 13021, + "stuttgart": 13022, + "curator": 13023, + "recruit": 13024, + "grocery": 13025, + "sympathetic": 13026, + "##tting": 13027, + "##fort": 13028, + "127": 13029, + "lotus": 13030, + "randolph": 13031, + "ancestor": 13032, + "##rand": 13033, + "succeeding": 13034, + "jupiter": 13035, + "1798": 13036, + "macedonian": 13037, + "##heads": 13038, + "hiking": 13039, + "1808": 13040, + "handing": 13041, + "fischer": 13042, + "##itive": 13043, + "garbage": 13044, + "node": 13045, + "##pies": 13046, + "prone": 13047, + "singular": 13048, + "papua": 13049, + "inclined": 13050, + "attractions": 13051, + "italia": 13052, + "pouring": 13053, + "motioned": 13054, + "grandma": 13055, + "garnered": 13056, + "jacksonville": 13057, + "corp": 13058, + "ego": 13059, + "ringing": 13060, + "aluminum": 13061, + "##hausen": 13062, + "ordering": 13063, + "##foot": 13064, + "drawer": 13065, + "traders": 13066, + "synagogue": 13067, + "##play": 13068, + "##kawa": 13069, + "resistant": 13070, + "wandering": 13071, + "fragile": 13072, + "fiona": 13073, + "teased": 13074, + "var": 13075, + "hardcore": 13076, + "soaked": 13077, + "jubilee": 13078, + "decisive": 13079, + "exposition": 13080, + "mercer": 13081, + "poster": 13082, + "valencia": 13083, + "hale": 13084, + "kuwait": 13085, + "1811": 13086, + "##ises": 13087, + "##wr": 13088, + "##eed": 13089, + "tavern": 13090, + "gamma": 13091, + "122": 13092, + "johan": 13093, + "##uer": 13094, + "airways": 13095, + "amino": 13096, + "gil": 13097, + "##ury": 13098, + "vocational": 13099, + "domains": 13100, + "torres": 13101, + "##sp": 13102, + "generator": 13103, + "folklore": 13104, + "outcomes": 13105, + "##keeper": 13106, + "canberra": 13107, + "shooter": 13108, + "fl": 13109, + "beams": 13110, + "confrontation": 13111, + "##lling": 13112, + "##gram": 13113, + "feb": 13114, + "aligned": 13115, + "forestry": 13116, + "pipeline": 13117, + "jax": 13118, + "motorway": 13119, + "conception": 13120, + "decay": 13121, + "##tos": 13122, + "coffin": 13123, + "##cott": 13124, + "stalin": 13125, + "1805": 13126, + "escorted": 13127, + "minded": 13128, + "##nam": 13129, + "sitcom": 13130, + "purchasing": 13131, + "twilight": 13132, + "veronica": 13133, + "additions": 13134, + "passive": 13135, + "tensions": 13136, + "straw": 13137, + "123": 13138, + "frequencies": 13139, + "1804": 13140, + "refugee": 13141, + "cultivation": 13142, + "##iate": 13143, + "christie": 13144, + "clary": 13145, + "bulletin": 13146, + "crept": 13147, + "disposal": 13148, + "##rich": 13149, + "##zong": 13150, + "processor": 13151, + "crescent": 13152, + "##rol": 13153, + "bmw": 13154, + "emphasized": 13155, + "whale": 13156, + "nazis": 13157, + "aurora": 13158, + "##eng": 13159, + "dwelling": 13160, + "hauled": 13161, + "sponsors": 13162, + "toledo": 13163, + "mega": 13164, + "ideology": 13165, + "theatres": 13166, + "tessa": 13167, + "cerambycidae": 13168, + "saves": 13169, + "turtle": 13170, + "cone": 13171, + "suspects": 13172, + "kara": 13173, + "rusty": 13174, + "yelling": 13175, + "greeks": 13176, + "mozart": 13177, + "shades": 13178, + "cocked": 13179, + "participant": 13180, + "##tro": 13181, + "shire": 13182, + "spit": 13183, + "freeze": 13184, + "necessity": 13185, + "##cos": 13186, + "inmates": 13187, + "nielsen": 13188, + "councillors": 13189, + "loaned": 13190, + "uncommon": 13191, + "omar": 13192, + "peasants": 13193, + "botanical": 13194, + "offspring": 13195, + "daniels": 13196, + "formations": 13197, + "jokes": 13198, + "1794": 13199, + "pioneers": 13200, + "sigma": 13201, + "licensing": 13202, + "##sus": 13203, + "wheelchair": 13204, + "polite": 13205, + "1807": 13206, + "liquor": 13207, + "pratt": 13208, + "trustee": 13209, + "##uta": 13210, + "forewings": 13211, + "balloon": 13212, + "##zz": 13213, + "kilometre": 13214, + "camping": 13215, + "explicit": 13216, + "casually": 13217, + "shawn": 13218, + "foolish": 13219, + "teammates": 13220, + "nm": 13221, + "hassan": 13222, + "carrie": 13223, + "judged": 13224, + "satisfy": 13225, + "vanessa": 13226, + "knives": 13227, + "selective": 13228, + "cnn": 13229, + "flowed": 13230, + "##lice": 13231, + "eclipse": 13232, + "stressed": 13233, + "eliza": 13234, + "mathematician": 13235, + "cease": 13236, + "cultivated": 13237, + "##roy": 13238, + "commissions": 13239, + "browns": 13240, + "##ania": 13241, + "destroyers": 13242, + "sheridan": 13243, + "meadow": 13244, + "##rius": 13245, + "minerals": 13246, + "##cial": 13247, + "downstream": 13248, + "clash": 13249, + "gram": 13250, + "memoirs": 13251, + "ventures": 13252, + "baha": 13253, + "seymour": 13254, + "archie": 13255, + "midlands": 13256, + "edith": 13257, + "fare": 13258, + "flynn": 13259, + "invite": 13260, + "canceled": 13261, + "tiles": 13262, + "stabbed": 13263, + "boulder": 13264, + "incorporate": 13265, + "amended": 13266, + "camden": 13267, + "facial": 13268, + "mollusk": 13269, + "unreleased": 13270, + "descriptions": 13271, + "yoga": 13272, + "grabs": 13273, + "550": 13274, + "raises": 13275, + "ramp": 13276, + "shiver": 13277, + "##rose": 13278, + "coined": 13279, + "pioneering": 13280, + "tunes": 13281, + "qing": 13282, + "warwick": 13283, + "tops": 13284, + "119": 13285, + "melanie": 13286, + "giles": 13287, + "##rous": 13288, + "wandered": 13289, + "##inal": 13290, + "annexed": 13291, + "nov": 13292, + "30th": 13293, + "unnamed": 13294, + "##ished": 13295, + "organizational": 13296, + "airplane": 13297, + "normandy": 13298, + "stoke": 13299, + "whistle": 13300, + "blessing": 13301, + "violations": 13302, + "chased": 13303, + "holders": 13304, + "shotgun": 13305, + "##ctic": 13306, + "outlet": 13307, + "reactor": 13308, + "##vik": 13309, + "tires": 13310, + "tearing": 13311, + "shores": 13312, + "fortified": 13313, + "mascot": 13314, + "constituencies": 13315, + "nc": 13316, + "columnist": 13317, + "productive": 13318, + "tibet": 13319, + "##rta": 13320, + "lineage": 13321, + "hooked": 13322, + "oct": 13323, + "tapes": 13324, + "judging": 13325, + "cody": 13326, + "##gger": 13327, + "hansen": 13328, + "kashmir": 13329, + "triggered": 13330, + "##eva": 13331, + "solved": 13332, + "cliffs": 13333, + "##tree": 13334, + "resisted": 13335, + "anatomy": 13336, + "protesters": 13337, + "transparent": 13338, + "implied": 13339, + "##iga": 13340, + "injection": 13341, + "mattress": 13342, + "excluding": 13343, + "##mbo": 13344, + "defenses": 13345, + "helpless": 13346, + "devotion": 13347, + "##elli": 13348, + "growl": 13349, + "liberals": 13350, + "weber": 13351, + "phenomena": 13352, + "atoms": 13353, + "plug": 13354, + "##iff": 13355, + "mortality": 13356, + "apprentice": 13357, + "howe": 13358, + "convincing": 13359, + "aaa": 13360, + "swimmer": 13361, + "barber": 13362, + "leone": 13363, + "promptly": 13364, + "sodium": 13365, + "def": 13366, + "nowadays": 13367, + "arise": 13368, + "##oning": 13369, + "gloucester": 13370, + "corrected": 13371, + "dignity": 13372, + "norm": 13373, + "erie": 13374, + "##ders": 13375, + "elders": 13376, + "evacuated": 13377, + "sylvia": 13378, + "compression": 13379, + "##yar": 13380, + "hartford": 13381, + "pose": 13382, + "backpack": 13383, + "reasoning": 13384, + "accepts": 13385, + "24th": 13386, + "wipe": 13387, + "millimetres": 13388, + "marcel": 13389, + "##oda": 13390, + "dodgers": 13391, + "albion": 13392, + "1790": 13393, + "overwhelmed": 13394, + "aerospace": 13395, + "oaks": 13396, + "1795": 13397, + "showcase": 13398, + "acknowledge": 13399, + "recovering": 13400, + "nolan": 13401, + "ashe": 13402, + "hurts": 13403, + "geology": 13404, + "fashioned": 13405, + "disappearance": 13406, + "farewell": 13407, + "swollen": 13408, + "shrug": 13409, + "marquis": 13410, + "wimbledon": 13411, + "124": 13412, + "rue": 13413, + "1792": 13414, + "commemorate": 13415, + "reduces": 13416, + "experiencing": 13417, + "inevitable": 13418, + "calcutta": 13419, + "intel": 13420, + "##court": 13421, + "murderer": 13422, + "sticking": 13423, + "fisheries": 13424, + "imagery": 13425, + "bloom": 13426, + "280": 13427, + "brake": 13428, + "##inus": 13429, + "gustav": 13430, + "hesitation": 13431, + "memorable": 13432, + "po": 13433, + "viral": 13434, + "beans": 13435, + "accidents": 13436, + "tunisia": 13437, + "antenna": 13438, + "spilled": 13439, + "consort": 13440, + "treatments": 13441, + "aye": 13442, + "perimeter": 13443, + "##gard": 13444, + "donation": 13445, + "hostage": 13446, + "migrated": 13447, + "banker": 13448, + "addiction": 13449, + "apex": 13450, + "lil": 13451, + "trout": 13452, + "##ously": 13453, + "conscience": 13454, + "##nova": 13455, + "rams": 13456, + "sands": 13457, + "genome": 13458, + "passionate": 13459, + "troubles": 13460, + "##lets": 13461, + "##set": 13462, + "amid": 13463, + "##ibility": 13464, + "##ret": 13465, + "higgins": 13466, + "exceed": 13467, + "vikings": 13468, + "##vie": 13469, + "payne": 13470, + "##zan": 13471, + "muscular": 13472, + "##ste": 13473, + "defendant": 13474, + "sucking": 13475, + "##wal": 13476, + "ibrahim": 13477, + "fuselage": 13478, + "claudia": 13479, + "vfl": 13480, + "europeans": 13481, + "snails": 13482, + "interval": 13483, + "##garh": 13484, + "preparatory": 13485, + "statewide": 13486, + "tasked": 13487, + "lacrosse": 13488, + "viktor": 13489, + "##lation": 13490, + "angola": 13491, + "##hra": 13492, + "flint": 13493, + "implications": 13494, + "employs": 13495, + "teens": 13496, + "patrons": 13497, + "stall": 13498, + "weekends": 13499, + "barriers": 13500, + "scrambled": 13501, + "nucleus": 13502, + "tehran": 13503, + "jenna": 13504, + "parsons": 13505, + "lifelong": 13506, + "robots": 13507, + "displacement": 13508, + "5000": 13509, + "##bles": 13510, + "precipitation": 13511, + "##gt": 13512, + "knuckles": 13513, + "clutched": 13514, + "1802": 13515, + "marrying": 13516, + "ecology": 13517, + "marx": 13518, + "accusations": 13519, + "declare": 13520, + "scars": 13521, + "kolkata": 13522, + "mat": 13523, + "meadows": 13524, + "bermuda": 13525, + "skeleton": 13526, + "finalists": 13527, + "vintage": 13528, + "crawl": 13529, + "coordinate": 13530, + "affects": 13531, + "subjected": 13532, + "orchestral": 13533, + "mistaken": 13534, + "##tc": 13535, + "mirrors": 13536, + "dipped": 13537, + "relied": 13538, + "260": 13539, + "arches": 13540, + "candle": 13541, + "##nick": 13542, + "incorporating": 13543, + "wildly": 13544, + "fond": 13545, + "basilica": 13546, + "owl": 13547, + "fringe": 13548, + "rituals": 13549, + "whispering": 13550, + "stirred": 13551, + "feud": 13552, + "tertiary": 13553, + "slick": 13554, + "goat": 13555, + "honorable": 13556, + "whereby": 13557, + "skip": 13558, + "ricardo": 13559, + "stripes": 13560, + "parachute": 13561, + "adjoining": 13562, + "submerged": 13563, + "synthesizer": 13564, + "##gren": 13565, + "intend": 13566, + "positively": 13567, + "ninety": 13568, + "phi": 13569, + "beaver": 13570, + "partition": 13571, + "fellows": 13572, + "alexis": 13573, + "prohibition": 13574, + "carlisle": 13575, + "bizarre": 13576, + "fraternity": 13577, + "##bre": 13578, + "doubts": 13579, + "icy": 13580, + "cbc": 13581, + "aquatic": 13582, + "sneak": 13583, + "sonny": 13584, + "combines": 13585, + "airports": 13586, + "crude": 13587, + "supervised": 13588, + "spatial": 13589, + "merge": 13590, + "alfonso": 13591, + "##bic": 13592, + "corrupt": 13593, + "scan": 13594, + "undergo": 13595, + "##ams": 13596, + "disabilities": 13597, + "colombian": 13598, + "comparing": 13599, + "dolphins": 13600, + "perkins": 13601, + "##lish": 13602, + "reprinted": 13603, + "unanimous": 13604, + "bounced": 13605, + "hairs": 13606, + "underworld": 13607, + "midwest": 13608, + "semester": 13609, + "bucket": 13610, + "paperback": 13611, + "miniseries": 13612, + "coventry": 13613, + "demise": 13614, + "##leigh": 13615, + "demonstrations": 13616, + "sensor": 13617, + "rotating": 13618, + "yan": 13619, + "##hler": 13620, + "arrange": 13621, + "soils": 13622, + "##idge": 13623, + "hyderabad": 13624, + "labs": 13625, + "##dr": 13626, + "brakes": 13627, + "grandchildren": 13628, + "##nde": 13629, + "negotiated": 13630, + "rover": 13631, + "ferrari": 13632, + "continuation": 13633, + "directorate": 13634, + "augusta": 13635, + "stevenson": 13636, + "counterpart": 13637, + "gore": 13638, + "##rda": 13639, + "nursery": 13640, + "rican": 13641, + "ave": 13642, + "collectively": 13643, + "broadly": 13644, + "pastoral": 13645, + "repertoire": 13646, + "asserted": 13647, + "discovering": 13648, + "nordic": 13649, + "styled": 13650, + "fiba": 13651, + "cunningham": 13652, + "harley": 13653, + "middlesex": 13654, + "survives": 13655, + "tumor": 13656, + "tempo": 13657, + "zack": 13658, + "aiming": 13659, + "lok": 13660, + "urgent": 13661, + "##rade": 13662, + "##nto": 13663, + "devils": 13664, + "##ement": 13665, + "contractor": 13666, + "turin": 13667, + "##wl": 13668, + "##ool": 13669, + "bliss": 13670, + "repaired": 13671, + "simmons": 13672, + "moan": 13673, + "astronomical": 13674, + "cr": 13675, + "negotiate": 13676, + "lyric": 13677, + "1890s": 13678, + "lara": 13679, + "bred": 13680, + "clad": 13681, + "angus": 13682, + "pbs": 13683, + "##ience": 13684, + "engineered": 13685, + "posed": 13686, + "##lk": 13687, + "hernandez": 13688, + "possessions": 13689, + "elbows": 13690, + "psychiatric": 13691, + "strokes": 13692, + "confluence": 13693, + "electorate": 13694, + "lifts": 13695, + "campuses": 13696, + "lava": 13697, + "alps": 13698, + "##ep": 13699, + "##ution": 13700, + "##date": 13701, + "physicist": 13702, + "woody": 13703, + "##page": 13704, + "##ographic": 13705, + "##itis": 13706, + "juliet": 13707, + "reformation": 13708, + "sparhawk": 13709, + "320": 13710, + "complement": 13711, + "suppressed": 13712, + "jewel": 13713, + "##½": 13714, + "floated": 13715, + "##kas": 13716, + "continuity": 13717, + "sadly": 13718, + "##ische": 13719, + "inability": 13720, + "melting": 13721, + "scanning": 13722, + "paula": 13723, + "flour": 13724, + "judaism": 13725, + "safer": 13726, + "vague": 13727, + "##lm": 13728, + "solving": 13729, + "curb": 13730, + "##stown": 13731, + "financially": 13732, + "gable": 13733, + "bees": 13734, + "expired": 13735, + "miserable": 13736, + "cassidy": 13737, + "dominion": 13738, + "1789": 13739, + "cupped": 13740, + "145": 13741, + "robbery": 13742, + "facto": 13743, + "amos": 13744, + "warden": 13745, + "resume": 13746, + "tallest": 13747, + "marvin": 13748, + "ing": 13749, + "pounded": 13750, + "usd": 13751, + "declaring": 13752, + "gasoline": 13753, + "##aux": 13754, + "darkened": 13755, + "270": 13756, + "650": 13757, + "sophomore": 13758, + "##mere": 13759, + "erection": 13760, + "gossip": 13761, + "televised": 13762, + "risen": 13763, + "dial": 13764, + "##eu": 13765, + "pillars": 13766, + "##link": 13767, + "passages": 13768, + "profound": 13769, + "##tina": 13770, + "arabian": 13771, + "ashton": 13772, + "silicon": 13773, + "nail": 13774, + "##ead": 13775, + "##lated": 13776, + "##wer": 13777, + "##hardt": 13778, + "fleming": 13779, + "firearms": 13780, + "ducked": 13781, + "circuits": 13782, + "blows": 13783, + "waterloo": 13784, + "titans": 13785, + "##lina": 13786, + "atom": 13787, + "fireplace": 13788, + "cheshire": 13789, + "financed": 13790, + "activation": 13791, + "algorithms": 13792, + "##zzi": 13793, + "constituent": 13794, + "catcher": 13795, + "cherokee": 13796, + "partnerships": 13797, + "sexuality": 13798, + "platoon": 13799, + "tragic": 13800, + "vivian": 13801, + "guarded": 13802, + "whiskey": 13803, + "meditation": 13804, + "poetic": 13805, + "##late": 13806, + "##nga": 13807, + "##ake": 13808, + "porto": 13809, + "listeners": 13810, + "dominance": 13811, + "kendra": 13812, + "mona": 13813, + "chandler": 13814, + "factions": 13815, + "22nd": 13816, + "salisbury": 13817, + "attitudes": 13818, + "derivative": 13819, + "##ido": 13820, + "##haus": 13821, + "intake": 13822, + "paced": 13823, + "javier": 13824, + "illustrator": 13825, + "barrels": 13826, + "bias": 13827, + "cockpit": 13828, + "burnett": 13829, + "dreamed": 13830, + "ensuing": 13831, + "##anda": 13832, + "receptors": 13833, + "someday": 13834, + "hawkins": 13835, + "mattered": 13836, + "##lal": 13837, + "slavic": 13838, + "1799": 13839, + "jesuit": 13840, + "cameroon": 13841, + "wasted": 13842, + "tai": 13843, + "wax": 13844, + "lowering": 13845, + "victorious": 13846, + "freaking": 13847, + "outright": 13848, + "hancock": 13849, + "librarian": 13850, + "sensing": 13851, + "bald": 13852, + "calcium": 13853, + "myers": 13854, + "tablet": 13855, + "announcing": 13856, + "barack": 13857, + "shipyard": 13858, + "pharmaceutical": 13859, + "##uan": 13860, + "greenwich": 13861, + "flush": 13862, + "medley": 13863, + "patches": 13864, + "wolfgang": 13865, + "pt": 13866, + "speeches": 13867, + "acquiring": 13868, + "exams": 13869, + "nikolai": 13870, + "##gg": 13871, + "hayden": 13872, + "kannada": 13873, + "##type": 13874, + "reilly": 13875, + "##pt": 13876, + "waitress": 13877, + "abdomen": 13878, + "devastated": 13879, + "capped": 13880, + "pseudonym": 13881, + "pharmacy": 13882, + "fulfill": 13883, + "paraguay": 13884, + "1796": 13885, + "clicked": 13886, + "##trom": 13887, + "archipelago": 13888, + "syndicated": 13889, + "##hman": 13890, + "lumber": 13891, + "orgasm": 13892, + "rejection": 13893, + "clifford": 13894, + "lorraine": 13895, + "advent": 13896, + "mafia": 13897, + "rodney": 13898, + "brock": 13899, + "##ght": 13900, + "##used": 13901, + "##elia": 13902, + "cassette": 13903, + "chamberlain": 13904, + "despair": 13905, + "mongolia": 13906, + "sensors": 13907, + "developmental": 13908, + "upstream": 13909, + "##eg": 13910, + "##alis": 13911, + "spanning": 13912, + "165": 13913, + "trombone": 13914, + "basque": 13915, + "seeded": 13916, + "interred": 13917, + "renewable": 13918, + "rhys": 13919, + "leapt": 13920, + "revision": 13921, + "molecule": 13922, + "##ages": 13923, + "chord": 13924, + "vicious": 13925, + "nord": 13926, + "shivered": 13927, + "23rd": 13928, + "arlington": 13929, + "debts": 13930, + "corpus": 13931, + "sunrise": 13932, + "bays": 13933, + "blackburn": 13934, + "centimetres": 13935, + "##uded": 13936, + "shuddered": 13937, + "gm": 13938, + "strangely": 13939, + "gripping": 13940, + "cartoons": 13941, + "isabelle": 13942, + "orbital": 13943, + "##ppa": 13944, + "seals": 13945, + "proving": 13946, + "##lton": 13947, + "refusal": 13948, + "strengthened": 13949, + "bust": 13950, + "assisting": 13951, + "baghdad": 13952, + "batsman": 13953, + "portrayal": 13954, + "mara": 13955, + "pushes": 13956, + "spears": 13957, + "og": 13958, + "##cock": 13959, + "reside": 13960, + "nathaniel": 13961, + "brennan": 13962, + "1776": 13963, + "confirmation": 13964, + "caucus": 13965, + "##worthy": 13966, + "markings": 13967, + "yemen": 13968, + "nobles": 13969, + "ku": 13970, + "lazy": 13971, + "viewer": 13972, + "catalan": 13973, + "encompasses": 13974, + "sawyer": 13975, + "##fall": 13976, + "sparked": 13977, + "substances": 13978, + "patents": 13979, + "braves": 13980, + "arranger": 13981, + "evacuation": 13982, + "sergio": 13983, + "persuade": 13984, + "dover": 13985, + "tolerance": 13986, + "penguin": 13987, + "cum": 13988, + "jockey": 13989, + "insufficient": 13990, + "townships": 13991, + "occupying": 13992, + "declining": 13993, + "plural": 13994, + "processed": 13995, + "projection": 13996, + "puppet": 13997, + "flanders": 13998, + "introduces": 13999, + "liability": 14000, + "##yon": 14001, + "gymnastics": 14002, + "antwerp": 14003, + "taipei": 14004, + "hobart": 14005, + "candles": 14006, + "jeep": 14007, + "wes": 14008, + "observers": 14009, + "126": 14010, + "chaplain": 14011, + "bundle": 14012, + "glorious": 14013, + "##hine": 14014, + "hazel": 14015, + "flung": 14016, + "sol": 14017, + "excavations": 14018, + "dumped": 14019, + "stares": 14020, + "sh": 14021, + "bangalore": 14022, + "triangular": 14023, + "icelandic": 14024, + "intervals": 14025, + "expressing": 14026, + "turbine": 14027, + "##vers": 14028, + "songwriting": 14029, + "crafts": 14030, + "##igo": 14031, + "jasmine": 14032, + "ditch": 14033, + "rite": 14034, + "##ways": 14035, + "entertaining": 14036, + "comply": 14037, + "sorrow": 14038, + "wrestlers": 14039, + "basel": 14040, + "emirates": 14041, + "marian": 14042, + "rivera": 14043, + "helpful": 14044, + "##some": 14045, + "caution": 14046, + "downward": 14047, + "networking": 14048, + "##atory": 14049, + "##tered": 14050, + "darted": 14051, + "genocide": 14052, + "emergence": 14053, + "replies": 14054, + "specializing": 14055, + "spokesman": 14056, + "convenient": 14057, + "unlocked": 14058, + "fading": 14059, + "augustine": 14060, + "concentrations": 14061, + "resemblance": 14062, + "elijah": 14063, + "investigator": 14064, + "andhra": 14065, + "##uda": 14066, + "promotes": 14067, + "bean": 14068, + "##rrell": 14069, + "fleeing": 14070, + "wan": 14071, + "simone": 14072, + "announcer": 14073, + "##ame": 14074, + "##bby": 14075, + "lydia": 14076, + "weaver": 14077, + "132": 14078, + "residency": 14079, + "modification": 14080, + "##fest": 14081, + "stretches": 14082, + "##ast": 14083, + "alternatively": 14084, + "nat": 14085, + "lowe": 14086, + "lacks": 14087, + "##ented": 14088, + "pam": 14089, + "tile": 14090, + "concealed": 14091, + "inferior": 14092, + "abdullah": 14093, + "residences": 14094, + "tissues": 14095, + "vengeance": 14096, + "##ided": 14097, + "moisture": 14098, + "peculiar": 14099, + "groove": 14100, + "zip": 14101, + "bologna": 14102, + "jennings": 14103, + "ninja": 14104, + "oversaw": 14105, + "zombies": 14106, + "pumping": 14107, + "batch": 14108, + "livingston": 14109, + "emerald": 14110, + "installations": 14111, + "1797": 14112, + "peel": 14113, + "nitrogen": 14114, + "rama": 14115, + "##fying": 14116, + "##star": 14117, + "schooling": 14118, + "strands": 14119, + "responding": 14120, + "werner": 14121, + "##ost": 14122, + "lime": 14123, + "casa": 14124, + "accurately": 14125, + "targeting": 14126, + "##rod": 14127, + "underway": 14128, + "##uru": 14129, + "hemisphere": 14130, + "lester": 14131, + "##yard": 14132, + "occupies": 14133, + "2d": 14134, + "griffith": 14135, + "angrily": 14136, + "reorganized": 14137, + "##owing": 14138, + "courtney": 14139, + "deposited": 14140, + "##dd": 14141, + "##30": 14142, + "estadio": 14143, + "##ifies": 14144, + "dunn": 14145, + "exiled": 14146, + "##ying": 14147, + "checks": 14148, + "##combe": 14149, + "##о": 14150, + "##fly": 14151, + "successes": 14152, + "unexpectedly": 14153, + "blu": 14154, + "assessed": 14155, + "##flower": 14156, + "##ه": 14157, + "observing": 14158, + "sacked": 14159, + "spiders": 14160, + "kn": 14161, + "##tail": 14162, + "mu": 14163, + "nodes": 14164, + "prosperity": 14165, + "audrey": 14166, + "divisional": 14167, + "155": 14168, + "broncos": 14169, + "tangled": 14170, + "adjust": 14171, + "feeds": 14172, + "erosion": 14173, + "paolo": 14174, + "surf": 14175, + "directory": 14176, + "snatched": 14177, + "humid": 14178, + "admiralty": 14179, + "screwed": 14180, + "gt": 14181, + "reddish": 14182, + "##nese": 14183, + "modules": 14184, + "trench": 14185, + "lamps": 14186, + "bind": 14187, + "leah": 14188, + "bucks": 14189, + "competes": 14190, + "##nz": 14191, + "##form": 14192, + "transcription": 14193, + "##uc": 14194, + "isles": 14195, + "violently": 14196, + "clutching": 14197, + "pga": 14198, + "cyclist": 14199, + "inflation": 14200, + "flats": 14201, + "ragged": 14202, + "unnecessary": 14203, + "##hian": 14204, + "stubborn": 14205, + "coordinated": 14206, + "harriet": 14207, + "baba": 14208, + "disqualified": 14209, + "330": 14210, + "insect": 14211, + "wolfe": 14212, + "##fies": 14213, + "reinforcements": 14214, + "rocked": 14215, + "duel": 14216, + "winked": 14217, + "embraced": 14218, + "bricks": 14219, + "##raj": 14220, + "hiatus": 14221, + "defeats": 14222, + "pending": 14223, + "brightly": 14224, + "jealousy": 14225, + "##xton": 14226, + "##hm": 14227, + "##uki": 14228, + "lena": 14229, + "gdp": 14230, + "colorful": 14231, + "##dley": 14232, + "stein": 14233, + "kidney": 14234, + "##shu": 14235, + "underwear": 14236, + "wanderers": 14237, + "##haw": 14238, + "##icus": 14239, + "guardians": 14240, + "m³": 14241, + "roared": 14242, + "habits": 14243, + "##wise": 14244, + "permits": 14245, + "gp": 14246, + "uranium": 14247, + "punished": 14248, + "disguise": 14249, + "bundesliga": 14250, + "elise": 14251, + "dundee": 14252, + "erotic": 14253, + "partisan": 14254, + "pi": 14255, + "collectors": 14256, + "float": 14257, + "individually": 14258, + "rendering": 14259, + "behavioral": 14260, + "bucharest": 14261, + "ser": 14262, + "hare": 14263, + "valerie": 14264, + "corporal": 14265, + "nutrition": 14266, + "proportional": 14267, + "##isa": 14268, + "immense": 14269, + "##kis": 14270, + "pavement": 14271, + "##zie": 14272, + "##eld": 14273, + "sutherland": 14274, + "crouched": 14275, + "1775": 14276, + "##lp": 14277, + "suzuki": 14278, + "trades": 14279, + "endurance": 14280, + "operas": 14281, + "crosby": 14282, + "prayed": 14283, + "priory": 14284, + "rory": 14285, + "socially": 14286, + "##urn": 14287, + "gujarat": 14288, + "##pu": 14289, + "walton": 14290, + "cube": 14291, + "pasha": 14292, + "privilege": 14293, + "lennon": 14294, + "floods": 14295, + "thorne": 14296, + "waterfall": 14297, + "nipple": 14298, + "scouting": 14299, + "approve": 14300, + "##lov": 14301, + "minorities": 14302, + "voter": 14303, + "dwight": 14304, + "extensions": 14305, + "assure": 14306, + "ballroom": 14307, + "slap": 14308, + "dripping": 14309, + "privileges": 14310, + "rejoined": 14311, + "confessed": 14312, + "demonstrating": 14313, + "patriotic": 14314, + "yell": 14315, + "investor": 14316, + "##uth": 14317, + "pagan": 14318, + "slumped": 14319, + "squares": 14320, + "##cle": 14321, + "##kins": 14322, + "confront": 14323, + "bert": 14324, + "embarrassment": 14325, + "##aid": 14326, + "aston": 14327, + "urging": 14328, + "sweater": 14329, + "starr": 14330, + "yuri": 14331, + "brains": 14332, + "williamson": 14333, + "commuter": 14334, + "mortar": 14335, + "structured": 14336, + "selfish": 14337, + "exports": 14338, + "##jon": 14339, + "cds": 14340, + "##him": 14341, + "unfinished": 14342, + "##rre": 14343, + "mortgage": 14344, + "destinations": 14345, + "##nagar": 14346, + "canoe": 14347, + "solitary": 14348, + "buchanan": 14349, + "delays": 14350, + "magistrate": 14351, + "fk": 14352, + "##pling": 14353, + "motivation": 14354, + "##lier": 14355, + "##vier": 14356, + "recruiting": 14357, + "assess": 14358, + "##mouth": 14359, + "malik": 14360, + "antique": 14361, + "1791": 14362, + "pius": 14363, + "rahman": 14364, + "reich": 14365, + "tub": 14366, + "zhou": 14367, + "smashed": 14368, + "airs": 14369, + "galway": 14370, + "xii": 14371, + "conditioning": 14372, + "honduras": 14373, + "discharged": 14374, + "dexter": 14375, + "##pf": 14376, + "lionel": 14377, + "129": 14378, + "debates": 14379, + "lemon": 14380, + "tiffany": 14381, + "volunteered": 14382, + "dom": 14383, + "dioxide": 14384, + "procession": 14385, + "devi": 14386, + "sic": 14387, + "tremendous": 14388, + "advertisements": 14389, + "colts": 14390, + "transferring": 14391, + "verdict": 14392, + "hanover": 14393, + "decommissioned": 14394, + "utter": 14395, + "relate": 14396, + "pac": 14397, + "racism": 14398, + "##top": 14399, + "beacon": 14400, + "limp": 14401, + "similarity": 14402, + "terra": 14403, + "occurrence": 14404, + "ant": 14405, + "##how": 14406, + "becky": 14407, + "capt": 14408, + "updates": 14409, + "armament": 14410, + "richie": 14411, + "pal": 14412, + "##graph": 14413, + "halloween": 14414, + "mayo": 14415, + "##ssen": 14416, + "##bone": 14417, + "cara": 14418, + "serena": 14419, + "fcc": 14420, + "dolls": 14421, + "obligations": 14422, + "##dling": 14423, + "violated": 14424, + "lafayette": 14425, + "jakarta": 14426, + "exploitation": 14427, + "##ime": 14428, + "infamous": 14429, + "iconic": 14430, + "##lah": 14431, + "##park": 14432, + "kitty": 14433, + "moody": 14434, + "reginald": 14435, + "dread": 14436, + "spill": 14437, + "crystals": 14438, + "olivier": 14439, + "modeled": 14440, + "bluff": 14441, + "equilibrium": 14442, + "separating": 14443, + "notices": 14444, + "ordnance": 14445, + "extinction": 14446, + "onset": 14447, + "cosmic": 14448, + "attachment": 14449, + "sammy": 14450, + "expose": 14451, + "privy": 14452, + "anchored": 14453, + "##bil": 14454, + "abbott": 14455, + "admits": 14456, + "bending": 14457, + "baritone": 14458, + "emmanuel": 14459, + "policeman": 14460, + "vaughan": 14461, + "winged": 14462, + "climax": 14463, + "dresses": 14464, + "denny": 14465, + "polytechnic": 14466, + "mohamed": 14467, + "burmese": 14468, + "authentic": 14469, + "nikki": 14470, + "genetics": 14471, + "grandparents": 14472, + "homestead": 14473, + "gaza": 14474, + "postponed": 14475, + "metacritic": 14476, + "una": 14477, + "##sby": 14478, + "##bat": 14479, + "unstable": 14480, + "dissertation": 14481, + "##rial": 14482, + "##cian": 14483, + "curls": 14484, + "obscure": 14485, + "uncovered": 14486, + "bronx": 14487, + "praying": 14488, + "disappearing": 14489, + "##hoe": 14490, + "prehistoric": 14491, + "coke": 14492, + "turret": 14493, + "mutations": 14494, + "nonprofit": 14495, + "pits": 14496, + "monaco": 14497, + "##ي": 14498, + "##usion": 14499, + "prominently": 14500, + "dispatched": 14501, + "podium": 14502, + "##mir": 14503, + "uci": 14504, + "##uation": 14505, + "133": 14506, + "fortifications": 14507, + "birthplace": 14508, + "kendall": 14509, + "##lby": 14510, + "##oll": 14511, + "preacher": 14512, + "rack": 14513, + "goodman": 14514, + "##rman": 14515, + "persistent": 14516, + "##ott": 14517, + "countless": 14518, + "jaime": 14519, + "recorder": 14520, + "lexington": 14521, + "persecution": 14522, + "jumps": 14523, + "renewal": 14524, + "wagons": 14525, + "##11": 14526, + "crushing": 14527, + "##holder": 14528, + "decorations": 14529, + "##lake": 14530, + "abundance": 14531, + "wrath": 14532, + "laundry": 14533, + "£1": 14534, + "garde": 14535, + "##rp": 14536, + "jeanne": 14537, + "beetles": 14538, + "peasant": 14539, + "##sl": 14540, + "splitting": 14541, + "caste": 14542, + "sergei": 14543, + "##rer": 14544, + "##ema": 14545, + "scripts": 14546, + "##ively": 14547, + "rub": 14548, + "satellites": 14549, + "##vor": 14550, + "inscribed": 14551, + "verlag": 14552, + "scrapped": 14553, + "gale": 14554, + "packages": 14555, + "chick": 14556, + "potato": 14557, + "slogan": 14558, + "kathleen": 14559, + "arabs": 14560, + "##culture": 14561, + "counterparts": 14562, + "reminiscent": 14563, + "choral": 14564, + "##tead": 14565, + "rand": 14566, + "retains": 14567, + "bushes": 14568, + "dane": 14569, + "accomplish": 14570, + "courtesy": 14571, + "closes": 14572, + "##oth": 14573, + "slaughter": 14574, + "hague": 14575, + "krakow": 14576, + "lawson": 14577, + "tailed": 14578, + "elias": 14579, + "ginger": 14580, + "##ttes": 14581, + "canopy": 14582, + "betrayal": 14583, + "rebuilding": 14584, + "turf": 14585, + "##hof": 14586, + "frowning": 14587, + "allegiance": 14588, + "brigades": 14589, + "kicks": 14590, + "rebuild": 14591, + "polls": 14592, + "alias": 14593, + "nationalism": 14594, + "td": 14595, + "rowan": 14596, + "audition": 14597, + "bowie": 14598, + "fortunately": 14599, + "recognizes": 14600, + "harp": 14601, + "dillon": 14602, + "horrified": 14603, + "##oro": 14604, + "renault": 14605, + "##tics": 14606, + "ropes": 14607, + "##α": 14608, + "presumed": 14609, + "rewarded": 14610, + "infrared": 14611, + "wiping": 14612, + "accelerated": 14613, + "illustration": 14614, + "##rid": 14615, + "presses": 14616, + "practitioners": 14617, + "badminton": 14618, + "##iard": 14619, + "detained": 14620, + "##tera": 14621, + "recognizing": 14622, + "relates": 14623, + "misery": 14624, + "##sies": 14625, + "##tly": 14626, + "reproduction": 14627, + "piercing": 14628, + "potatoes": 14629, + "thornton": 14630, + "esther": 14631, + "manners": 14632, + "hbo": 14633, + "##aan": 14634, + "ours": 14635, + "bullshit": 14636, + "ernie": 14637, + "perennial": 14638, + "sensitivity": 14639, + "illuminated": 14640, + "rupert": 14641, + "##jin": 14642, + "##iss": 14643, + "##ear": 14644, + "rfc": 14645, + "nassau": 14646, + "##dock": 14647, + "staggered": 14648, + "socialism": 14649, + "##haven": 14650, + "appointments": 14651, + "nonsense": 14652, + "prestige": 14653, + "sharma": 14654, + "haul": 14655, + "##tical": 14656, + "solidarity": 14657, + "gps": 14658, + "##ook": 14659, + "##rata": 14660, + "igor": 14661, + "pedestrian": 14662, + "##uit": 14663, + "baxter": 14664, + "tenants": 14665, + "wires": 14666, + "medication": 14667, + "unlimited": 14668, + "guiding": 14669, + "impacts": 14670, + "diabetes": 14671, + "##rama": 14672, + "sasha": 14673, + "pas": 14674, + "clive": 14675, + "extraction": 14676, + "131": 14677, + "continually": 14678, + "constraints": 14679, + "##bilities": 14680, + "sonata": 14681, + "hunted": 14682, + "sixteenth": 14683, + "chu": 14684, + "planting": 14685, + "quote": 14686, + "mayer": 14687, + "pretended": 14688, + "abs": 14689, + "spat": 14690, + "##hua": 14691, + "ceramic": 14692, + "##cci": 14693, + "curtains": 14694, + "pigs": 14695, + "pitching": 14696, + "##dad": 14697, + "latvian": 14698, + "sore": 14699, + "dayton": 14700, + "##sted": 14701, + "##qi": 14702, + "patrols": 14703, + "slice": 14704, + "playground": 14705, + "##nted": 14706, + "shone": 14707, + "stool": 14708, + "apparatus": 14709, + "inadequate": 14710, + "mates": 14711, + "treason": 14712, + "##ija": 14713, + "desires": 14714, + "##liga": 14715, + "##croft": 14716, + "somalia": 14717, + "laurent": 14718, + "mir": 14719, + "leonardo": 14720, + "oracle": 14721, + "grape": 14722, + "obliged": 14723, + "chevrolet": 14724, + "thirteenth": 14725, + "stunning": 14726, + "enthusiastic": 14727, + "##ede": 14728, + "accounted": 14729, + "concludes": 14730, + "currents": 14731, + "basil": 14732, + "##kovic": 14733, + "drought": 14734, + "##rica": 14735, + "mai": 14736, + "##aire": 14737, + "shove": 14738, + "posting": 14739, + "##shed": 14740, + "pilgrimage": 14741, + "humorous": 14742, + "packing": 14743, + "fry": 14744, + "pencil": 14745, + "wines": 14746, + "smells": 14747, + "144": 14748, + "marilyn": 14749, + "aching": 14750, + "newest": 14751, + "clung": 14752, + "bon": 14753, + "neighbours": 14754, + "sanctioned": 14755, + "##pie": 14756, + "mug": 14757, + "##stock": 14758, + "drowning": 14759, + "##mma": 14760, + "hydraulic": 14761, + "##vil": 14762, + "hiring": 14763, + "reminder": 14764, + "lilly": 14765, + "investigators": 14766, + "##ncies": 14767, + "sour": 14768, + "##eous": 14769, + "compulsory": 14770, + "packet": 14771, + "##rion": 14772, + "##graphic": 14773, + "##elle": 14774, + "cannes": 14775, + "##inate": 14776, + "depressed": 14777, + "##rit": 14778, + "heroic": 14779, + "importantly": 14780, + "theresa": 14781, + "##tled": 14782, + "conway": 14783, + "saturn": 14784, + "marginal": 14785, + "rae": 14786, + "##xia": 14787, + "corresponds": 14788, + "royce": 14789, + "pact": 14790, + "jasper": 14791, + "explosives": 14792, + "packaging": 14793, + "aluminium": 14794, + "##ttered": 14795, + "denotes": 14796, + "rhythmic": 14797, + "spans": 14798, + "assignments": 14799, + "hereditary": 14800, + "outlined": 14801, + "originating": 14802, + "sundays": 14803, + "lad": 14804, + "reissued": 14805, + "greeting": 14806, + "beatrice": 14807, + "##dic": 14808, + "pillar": 14809, + "marcos": 14810, + "plots": 14811, + "handbook": 14812, + "alcoholic": 14813, + "judiciary": 14814, + "avant": 14815, + "slides": 14816, + "extract": 14817, + "masculine": 14818, + "blur": 14819, + "##eum": 14820, + "##force": 14821, + "homage": 14822, + "trembled": 14823, + "owens": 14824, + "hymn": 14825, + "trey": 14826, + "omega": 14827, + "signaling": 14828, + "socks": 14829, + "accumulated": 14830, + "reacted": 14831, + "attic": 14832, + "theo": 14833, + "lining": 14834, + "angie": 14835, + "distraction": 14836, + "primera": 14837, + "talbot": 14838, + "##key": 14839, + "1200": 14840, + "ti": 14841, + "creativity": 14842, + "billed": 14843, + "##hey": 14844, + "deacon": 14845, + "eduardo": 14846, + "identifies": 14847, + "proposition": 14848, + "dizzy": 14849, + "gunner": 14850, + "hogan": 14851, + "##yam": 14852, + "##pping": 14853, + "##hol": 14854, + "ja": 14855, + "##chan": 14856, + "jensen": 14857, + "reconstructed": 14858, + "##berger": 14859, + "clearance": 14860, + "darius": 14861, + "##nier": 14862, + "abe": 14863, + "harlem": 14864, + "plea": 14865, + "dei": 14866, + "circled": 14867, + "emotionally": 14868, + "notation": 14869, + "fascist": 14870, + "neville": 14871, + "exceeded": 14872, + "upwards": 14873, + "viable": 14874, + "ducks": 14875, + "##fo": 14876, + "workforce": 14877, + "racer": 14878, + "limiting": 14879, + "shri": 14880, + "##lson": 14881, + "possesses": 14882, + "1600": 14883, + "kerr": 14884, + "moths": 14885, + "devastating": 14886, + "laden": 14887, + "disturbing": 14888, + "locking": 14889, + "##cture": 14890, + "gal": 14891, + "fearing": 14892, + "accreditation": 14893, + "flavor": 14894, + "aide": 14895, + "1870s": 14896, + "mountainous": 14897, + "##baum": 14898, + "melt": 14899, + "##ures": 14900, + "motel": 14901, + "texture": 14902, + "servers": 14903, + "soda": 14904, + "##mb": 14905, + "herd": 14906, + "##nium": 14907, + "erect": 14908, + "puzzled": 14909, + "hum": 14910, + "peggy": 14911, + "examinations": 14912, + "gould": 14913, + "testified": 14914, + "geoff": 14915, + "ren": 14916, + "devised": 14917, + "sacks": 14918, + "##law": 14919, + "denial": 14920, + "posters": 14921, + "grunted": 14922, + "cesar": 14923, + "tutor": 14924, + "ec": 14925, + "gerry": 14926, + "offerings": 14927, + "byrne": 14928, + "falcons": 14929, + "combinations": 14930, + "ct": 14931, + "incoming": 14932, + "pardon": 14933, + "rocking": 14934, + "26th": 14935, + "avengers": 14936, + "flared": 14937, + "mankind": 14938, + "seller": 14939, + "uttar": 14940, + "loch": 14941, + "nadia": 14942, + "stroking": 14943, + "exposing": 14944, + "##hd": 14945, + "fertile": 14946, + "ancestral": 14947, + "instituted": 14948, + "##has": 14949, + "noises": 14950, + "prophecy": 14951, + "taxation": 14952, + "eminent": 14953, + "vivid": 14954, + "pol": 14955, + "##bol": 14956, + "dart": 14957, + "indirect": 14958, + "multimedia": 14959, + "notebook": 14960, + "upside": 14961, + "displaying": 14962, + "adrenaline": 14963, + "referenced": 14964, + "geometric": 14965, + "##iving": 14966, + "progression": 14967, + "##ddy": 14968, + "blunt": 14969, + "announce": 14970, + "##far": 14971, + "implementing": 14972, + "##lav": 14973, + "aggression": 14974, + "liaison": 14975, + "cooler": 14976, + "cares": 14977, + "headache": 14978, + "plantations": 14979, + "gorge": 14980, + "dots": 14981, + "impulse": 14982, + "thickness": 14983, + "ashamed": 14984, + "averaging": 14985, + "kathy": 14986, + "obligation": 14987, + "precursor": 14988, + "137": 14989, + "fowler": 14990, + "symmetry": 14991, + "thee": 14992, + "225": 14993, + "hears": 14994, + "##rai": 14995, + "undergoing": 14996, + "ads": 14997, + "butcher": 14998, + "bowler": 14999, + "##lip": 15000, + "cigarettes": 15001, + "subscription": 15002, + "goodness": 15003, + "##ically": 15004, + "browne": 15005, + "##hos": 15006, + "##tech": 15007, + "kyoto": 15008, + "donor": 15009, + "##erty": 15010, + "damaging": 15011, + "friction": 15012, + "drifting": 15013, + "expeditions": 15014, + "hardened": 15015, + "prostitution": 15016, + "152": 15017, + "fauna": 15018, + "blankets": 15019, + "claw": 15020, + "tossing": 15021, + "snarled": 15022, + "butterflies": 15023, + "recruits": 15024, + "investigative": 15025, + "coated": 15026, + "healed": 15027, + "138": 15028, + "communal": 15029, + "hai": 15030, + "xiii": 15031, + "academics": 15032, + "boone": 15033, + "psychologist": 15034, + "restless": 15035, + "lahore": 15036, + "stephens": 15037, + "mba": 15038, + "brendan": 15039, + "foreigners": 15040, + "printer": 15041, + "##pc": 15042, + "ached": 15043, + "explode": 15044, + "27th": 15045, + "deed": 15046, + "scratched": 15047, + "dared": 15048, + "##pole": 15049, + "cardiac": 15050, + "1780": 15051, + "okinawa": 15052, + "proto": 15053, + "commando": 15054, + "compelled": 15055, + "oddly": 15056, + "electrons": 15057, + "##base": 15058, + "replica": 15059, + "thanksgiving": 15060, + "##rist": 15061, + "sheila": 15062, + "deliberate": 15063, + "stafford": 15064, + "tidal": 15065, + "representations": 15066, + "hercules": 15067, + "ou": 15068, + "##path": 15069, + "##iated": 15070, + "kidnapping": 15071, + "lenses": 15072, + "##tling": 15073, + "deficit": 15074, + "samoa": 15075, + "mouths": 15076, + "consuming": 15077, + "computational": 15078, + "maze": 15079, + "granting": 15080, + "smirk": 15081, + "razor": 15082, + "fixture": 15083, + "ideals": 15084, + "inviting": 15085, + "aiden": 15086, + "nominal": 15087, + "##vs": 15088, + "issuing": 15089, + "julio": 15090, + "pitt": 15091, + "ramsey": 15092, + "docks": 15093, + "##oss": 15094, + "exhaust": 15095, + "##owed": 15096, + "bavarian": 15097, + "draped": 15098, + "anterior": 15099, + "mating": 15100, + "ethiopian": 15101, + "explores": 15102, + "noticing": 15103, + "##nton": 15104, + "discarded": 15105, + "convenience": 15106, + "hoffman": 15107, + "endowment": 15108, + "beasts": 15109, + "cartridge": 15110, + "mormon": 15111, + "paternal": 15112, + "probe": 15113, + "sleeves": 15114, + "interfere": 15115, + "lump": 15116, + "deadline": 15117, + "##rail": 15118, + "jenks": 15119, + "bulldogs": 15120, + "scrap": 15121, + "alternating": 15122, + "justified": 15123, + "reproductive": 15124, + "nam": 15125, + "seize": 15126, + "descending": 15127, + "secretariat": 15128, + "kirby": 15129, + "coupe": 15130, + "grouped": 15131, + "smash": 15132, + "panther": 15133, + "sedan": 15134, + "tapping": 15135, + "##18": 15136, + "lola": 15137, + "cheer": 15138, + "germanic": 15139, + "unfortunate": 15140, + "##eter": 15141, + "unrelated": 15142, + "##fan": 15143, + "subordinate": 15144, + "##sdale": 15145, + "suzanne": 15146, + "advertisement": 15147, + "##ility": 15148, + "horsepower": 15149, + "##lda": 15150, + "cautiously": 15151, + "discourse": 15152, + "luigi": 15153, + "##mans": 15154, + "##fields": 15155, + "noun": 15156, + "prevalent": 15157, + "mao": 15158, + "schneider": 15159, + "everett": 15160, + "surround": 15161, + "governorate": 15162, + "kira": 15163, + "##avia": 15164, + "westward": 15165, + "##take": 15166, + "misty": 15167, + "rails": 15168, + "sustainability": 15169, + "134": 15170, + "unused": 15171, + "##rating": 15172, + "packs": 15173, + "toast": 15174, + "unwilling": 15175, + "regulate": 15176, + "thy": 15177, + "suffrage": 15178, + "nile": 15179, + "awe": 15180, + "assam": 15181, + "definitions": 15182, + "travelers": 15183, + "affordable": 15184, + "##rb": 15185, + "conferred": 15186, + "sells": 15187, + "undefeated": 15188, + "beneficial": 15189, + "torso": 15190, + "basal": 15191, + "repeating": 15192, + "remixes": 15193, + "##pass": 15194, + "bahrain": 15195, + "cables": 15196, + "fang": 15197, + "##itated": 15198, + "excavated": 15199, + "numbering": 15200, + "statutory": 15201, + "##rey": 15202, + "deluxe": 15203, + "##lian": 15204, + "forested": 15205, + "ramirez": 15206, + "derbyshire": 15207, + "zeus": 15208, + "slamming": 15209, + "transfers": 15210, + "astronomer": 15211, + "banana": 15212, + "lottery": 15213, + "berg": 15214, + "histories": 15215, + "bamboo": 15216, + "##uchi": 15217, + "resurrection": 15218, + "posterior": 15219, + "bowls": 15220, + "vaguely": 15221, + "##thi": 15222, + "thou": 15223, + "preserving": 15224, + "tensed": 15225, + "offence": 15226, + "##inas": 15227, + "meyrick": 15228, + "callum": 15229, + "ridden": 15230, + "watt": 15231, + "langdon": 15232, + "tying": 15233, + "lowland": 15234, + "snorted": 15235, + "daring": 15236, + "truman": 15237, + "##hale": 15238, + "##girl": 15239, + "aura": 15240, + "overly": 15241, + "filing": 15242, + "weighing": 15243, + "goa": 15244, + "infections": 15245, + "philanthropist": 15246, + "saunders": 15247, + "eponymous": 15248, + "##owski": 15249, + "latitude": 15250, + "perspectives": 15251, + "reviewing": 15252, + "mets": 15253, + "commandant": 15254, + "radial": 15255, + "##kha": 15256, + "flashlight": 15257, + "reliability": 15258, + "koch": 15259, + "vowels": 15260, + "amazed": 15261, + "ada": 15262, + "elaine": 15263, + "supper": 15264, + "##rth": 15265, + "##encies": 15266, + "predator": 15267, + "debated": 15268, + "soviets": 15269, + "cola": 15270, + "##boards": 15271, + "##nah": 15272, + "compartment": 15273, + "crooked": 15274, + "arbitrary": 15275, + "fourteenth": 15276, + "##ctive": 15277, + "havana": 15278, + "majors": 15279, + "steelers": 15280, + "clips": 15281, + "profitable": 15282, + "ambush": 15283, + "exited": 15284, + "packers": 15285, + "##tile": 15286, + "nude": 15287, + "cracks": 15288, + "fungi": 15289, + "##е": 15290, + "limb": 15291, + "trousers": 15292, + "josie": 15293, + "shelby": 15294, + "tens": 15295, + "frederic": 15296, + "##ος": 15297, + "definite": 15298, + "smoothly": 15299, + "constellation": 15300, + "insult": 15301, + "baton": 15302, + "discs": 15303, + "lingering": 15304, + "##nco": 15305, + "conclusions": 15306, + "lent": 15307, + "staging": 15308, + "becker": 15309, + "grandpa": 15310, + "shaky": 15311, + "##tron": 15312, + "einstein": 15313, + "obstacles": 15314, + "sk": 15315, + "adverse": 15316, + "elle": 15317, + "economically": 15318, + "##moto": 15319, + "mccartney": 15320, + "thor": 15321, + "dismissal": 15322, + "motions": 15323, + "readings": 15324, + "nostrils": 15325, + "treatise": 15326, + "##pace": 15327, + "squeezing": 15328, + "evidently": 15329, + "prolonged": 15330, + "1783": 15331, + "venezuelan": 15332, + "je": 15333, + "marguerite": 15334, + "beirut": 15335, + "takeover": 15336, + "shareholders": 15337, + "##vent": 15338, + "denise": 15339, + "digit": 15340, + "airplay": 15341, + "norse": 15342, + "##bbling": 15343, + "imaginary": 15344, + "pills": 15345, + "hubert": 15346, + "blaze": 15347, + "vacated": 15348, + "eliminating": 15349, + "##ello": 15350, + "vine": 15351, + "mansfield": 15352, + "##tty": 15353, + "retrospective": 15354, + "barrow": 15355, + "borne": 15356, + "clutch": 15357, + "bail": 15358, + "forensic": 15359, + "weaving": 15360, + "##nett": 15361, + "##witz": 15362, + "desktop": 15363, + "citadel": 15364, + "promotions": 15365, + "worrying": 15366, + "dorset": 15367, + "ieee": 15368, + "subdivided": 15369, + "##iating": 15370, + "manned": 15371, + "expeditionary": 15372, + "pickup": 15373, + "synod": 15374, + "chuckle": 15375, + "185": 15376, + "barney": 15377, + "##rz": 15378, + "##ffin": 15379, + "functionality": 15380, + "karachi": 15381, + "litigation": 15382, + "meanings": 15383, + "uc": 15384, + "lick": 15385, + "turbo": 15386, + "anders": 15387, + "##ffed": 15388, + "execute": 15389, + "curl": 15390, + "oppose": 15391, + "ankles": 15392, + "typhoon": 15393, + "##د": 15394, + "##ache": 15395, + "##asia": 15396, + "linguistics": 15397, + "compassion": 15398, + "pressures": 15399, + "grazing": 15400, + "perfection": 15401, + "##iting": 15402, + "immunity": 15403, + "monopoly": 15404, + "muddy": 15405, + "backgrounds": 15406, + "136": 15407, + "namibia": 15408, + "francesca": 15409, + "monitors": 15410, + "attracting": 15411, + "stunt": 15412, + "tuition": 15413, + "##ии": 15414, + "vegetable": 15415, + "##mates": 15416, + "##quent": 15417, + "mgm": 15418, + "jen": 15419, + "complexes": 15420, + "forts": 15421, + "##ond": 15422, + "cellar": 15423, + "bites": 15424, + "seventeenth": 15425, + "royals": 15426, + "flemish": 15427, + "failures": 15428, + "mast": 15429, + "charities": 15430, + "##cular": 15431, + "peruvian": 15432, + "capitals": 15433, + "macmillan": 15434, + "ipswich": 15435, + "outward": 15436, + "frigate": 15437, + "postgraduate": 15438, + "folds": 15439, + "employing": 15440, + "##ouse": 15441, + "concurrently": 15442, + "fiery": 15443, + "##tai": 15444, + "contingent": 15445, + "nightmares": 15446, + "monumental": 15447, + "nicaragua": 15448, + "##kowski": 15449, + "lizard": 15450, + "mal": 15451, + "fielding": 15452, + "gig": 15453, + "reject": 15454, + "##pad": 15455, + "harding": 15456, + "##ipe": 15457, + "coastline": 15458, + "##cin": 15459, + "##nos": 15460, + "beethoven": 15461, + "humphrey": 15462, + "innovations": 15463, + "##tam": 15464, + "##nge": 15465, + "norris": 15466, + "doris": 15467, + "solicitor": 15468, + "huang": 15469, + "obey": 15470, + "141": 15471, + "##lc": 15472, + "niagara": 15473, + "##tton": 15474, + "shelves": 15475, + "aug": 15476, + "bourbon": 15477, + "curry": 15478, + "nightclub": 15479, + "specifications": 15480, + "hilton": 15481, + "##ndo": 15482, + "centennial": 15483, + "dispersed": 15484, + "worm": 15485, + "neglected": 15486, + "briggs": 15487, + "sm": 15488, + "font": 15489, + "kuala": 15490, + "uneasy": 15491, + "plc": 15492, + "##nstein": 15493, + "##bound": 15494, + "##aking": 15495, + "##burgh": 15496, + "awaiting": 15497, + "pronunciation": 15498, + "##bbed": 15499, + "##quest": 15500, + "eh": 15501, + "optimal": 15502, + "zhu": 15503, + "raped": 15504, + "greens": 15505, + "presided": 15506, + "brenda": 15507, + "worries": 15508, + "##life": 15509, + "venetian": 15510, + "marxist": 15511, + "turnout": 15512, + "##lius": 15513, + "refined": 15514, + "braced": 15515, + "sins": 15516, + "grasped": 15517, + "sunderland": 15518, + "nickel": 15519, + "speculated": 15520, + "lowell": 15521, + "cyrillic": 15522, + "communism": 15523, + "fundraising": 15524, + "resembling": 15525, + "colonists": 15526, + "mutant": 15527, + "freddie": 15528, + "usc": 15529, + "##mos": 15530, + "gratitude": 15531, + "##run": 15532, + "mural": 15533, + "##lous": 15534, + "chemist": 15535, + "wi": 15536, + "reminds": 15537, + "28th": 15538, + "steals": 15539, + "tess": 15540, + "pietro": 15541, + "##ingen": 15542, + "promoter": 15543, + "ri": 15544, + "microphone": 15545, + "honoured": 15546, + "rai": 15547, + "sant": 15548, + "##qui": 15549, + "feather": 15550, + "##nson": 15551, + "burlington": 15552, + "kurdish": 15553, + "terrorists": 15554, + "deborah": 15555, + "sickness": 15556, + "##wed": 15557, + "##eet": 15558, + "hazard": 15559, + "irritated": 15560, + "desperation": 15561, + "veil": 15562, + "clarity": 15563, + "##rik": 15564, + "jewels": 15565, + "xv": 15566, + "##gged": 15567, + "##ows": 15568, + "##cup": 15569, + "berkshire": 15570, + "unfair": 15571, + "mysteries": 15572, + "orchid": 15573, + "winced": 15574, + "exhaustion": 15575, + "renovations": 15576, + "stranded": 15577, + "obe": 15578, + "infinity": 15579, + "##nies": 15580, + "adapt": 15581, + "redevelopment": 15582, + "thanked": 15583, + "registry": 15584, + "olga": 15585, + "domingo": 15586, + "noir": 15587, + "tudor": 15588, + "ole": 15589, + "##atus": 15590, + "commenting": 15591, + "behaviors": 15592, + "##ais": 15593, + "crisp": 15594, + "pauline": 15595, + "probable": 15596, + "stirling": 15597, + "wigan": 15598, + "##bian": 15599, + "paralympics": 15600, + "panting": 15601, + "surpassed": 15602, + "##rew": 15603, + "luca": 15604, + "barred": 15605, + "pony": 15606, + "famed": 15607, + "##sters": 15608, + "cassandra": 15609, + "waiter": 15610, + "carolyn": 15611, + "exported": 15612, + "##orted": 15613, + "andres": 15614, + "destructive": 15615, + "deeds": 15616, + "jonah": 15617, + "castles": 15618, + "vacancy": 15619, + "suv": 15620, + "##glass": 15621, + "1788": 15622, + "orchard": 15623, + "yep": 15624, + "famine": 15625, + "belarusian": 15626, + "sprang": 15627, + "##forth": 15628, + "skinny": 15629, + "##mis": 15630, + "administrators": 15631, + "rotterdam": 15632, + "zambia": 15633, + "zhao": 15634, + "boiler": 15635, + "discoveries": 15636, + "##ride": 15637, + "##physics": 15638, + "lucius": 15639, + "disappointing": 15640, + "outreach": 15641, + "spoon": 15642, + "##frame": 15643, + "qualifications": 15644, + "unanimously": 15645, + "enjoys": 15646, + "regency": 15647, + "##iidae": 15648, + "stade": 15649, + "realism": 15650, + "veterinary": 15651, + "rodgers": 15652, + "dump": 15653, + "alain": 15654, + "chestnut": 15655, + "castile": 15656, + "censorship": 15657, + "rumble": 15658, + "gibbs": 15659, + "##itor": 15660, + "communion": 15661, + "reggae": 15662, + "inactivated": 15663, + "logs": 15664, + "loads": 15665, + "##houses": 15666, + "homosexual": 15667, + "##iano": 15668, + "ale": 15669, + "informs": 15670, + "##cas": 15671, + "phrases": 15672, + "plaster": 15673, + "linebacker": 15674, + "ambrose": 15675, + "kaiser": 15676, + "fascinated": 15677, + "850": 15678, + "limerick": 15679, + "recruitment": 15680, + "forge": 15681, + "mastered": 15682, + "##nding": 15683, + "leinster": 15684, + "rooted": 15685, + "threaten": 15686, + "##strom": 15687, + "borneo": 15688, + "##hes": 15689, + "suggestions": 15690, + "scholarships": 15691, + "propeller": 15692, + "documentaries": 15693, + "patronage": 15694, + "coats": 15695, + "constructing": 15696, + "invest": 15697, + "neurons": 15698, + "comet": 15699, + "entirety": 15700, + "shouts": 15701, + "identities": 15702, + "annoying": 15703, + "unchanged": 15704, + "wary": 15705, + "##antly": 15706, + "##ogy": 15707, + "neat": 15708, + "oversight": 15709, + "##kos": 15710, + "phillies": 15711, + "replay": 15712, + "constance": 15713, + "##kka": 15714, + "incarnation": 15715, + "humble": 15716, + "skies": 15717, + "minus": 15718, + "##acy": 15719, + "smithsonian": 15720, + "##chel": 15721, + "guerrilla": 15722, + "jar": 15723, + "cadets": 15724, + "##plate": 15725, + "surplus": 15726, + "audit": 15727, + "##aru": 15728, + "cracking": 15729, + "joanna": 15730, + "louisa": 15731, + "pacing": 15732, + "##lights": 15733, + "intentionally": 15734, + "##iri": 15735, + "diner": 15736, + "nwa": 15737, + "imprint": 15738, + "australians": 15739, + "tong": 15740, + "unprecedented": 15741, + "bunker": 15742, + "naive": 15743, + "specialists": 15744, + "ark": 15745, + "nichols": 15746, + "railing": 15747, + "leaked": 15748, + "pedal": 15749, + "##uka": 15750, + "shrub": 15751, + "longing": 15752, + "roofs": 15753, + "v8": 15754, + "captains": 15755, + "neural": 15756, + "tuned": 15757, + "##ntal": 15758, + "##jet": 15759, + "emission": 15760, + "medina": 15761, + "frantic": 15762, + "codex": 15763, + "definitive": 15764, + "sid": 15765, + "abolition": 15766, + "intensified": 15767, + "stocks": 15768, + "enrique": 15769, + "sustain": 15770, + "genoa": 15771, + "oxide": 15772, + "##written": 15773, + "clues": 15774, + "cha": 15775, + "##gers": 15776, + "tributaries": 15777, + "fragment": 15778, + "venom": 15779, + "##rity": 15780, + "##ente": 15781, + "##sca": 15782, + "muffled": 15783, + "vain": 15784, + "sire": 15785, + "laos": 15786, + "##ingly": 15787, + "##hana": 15788, + "hastily": 15789, + "snapping": 15790, + "surfaced": 15791, + "sentiment": 15792, + "motive": 15793, + "##oft": 15794, + "contests": 15795, + "approximate": 15796, + "mesa": 15797, + "luckily": 15798, + "dinosaur": 15799, + "exchanges": 15800, + "propelled": 15801, + "accord": 15802, + "bourne": 15803, + "relieve": 15804, + "tow": 15805, + "masks": 15806, + "offended": 15807, + "##ues": 15808, + "cynthia": 15809, + "##mmer": 15810, + "rains": 15811, + "bartender": 15812, + "zinc": 15813, + "reviewers": 15814, + "lois": 15815, + "##sai": 15816, + "legged": 15817, + "arrogant": 15818, + "rafe": 15819, + "rosie": 15820, + "comprise": 15821, + "handicap": 15822, + "blockade": 15823, + "inlet": 15824, + "lagoon": 15825, + "copied": 15826, + "drilling": 15827, + "shelley": 15828, + "petals": 15829, + "##inian": 15830, + "mandarin": 15831, + "obsolete": 15832, + "##inated": 15833, + "onward": 15834, + "arguably": 15835, + "productivity": 15836, + "cindy": 15837, + "praising": 15838, + "seldom": 15839, + "busch": 15840, + "discusses": 15841, + "raleigh": 15842, + "shortage": 15843, + "ranged": 15844, + "stanton": 15845, + "encouragement": 15846, + "firstly": 15847, + "conceded": 15848, + "overs": 15849, + "temporal": 15850, + "##uke": 15851, + "cbe": 15852, + "##bos": 15853, + "woo": 15854, + "certainty": 15855, + "pumps": 15856, + "##pton": 15857, + "stalked": 15858, + "##uli": 15859, + "lizzie": 15860, + "periodic": 15861, + "thieves": 15862, + "weaker": 15863, + "##night": 15864, + "gases": 15865, + "shoving": 15866, + "chooses": 15867, + "wc": 15868, + "##chemical": 15869, + "prompting": 15870, + "weights": 15871, + "##kill": 15872, + "robust": 15873, + "flanked": 15874, + "sticky": 15875, + "hu": 15876, + "tuberculosis": 15877, + "##eb": 15878, + "##eal": 15879, + "christchurch": 15880, + "resembled": 15881, + "wallet": 15882, + "reese": 15883, + "inappropriate": 15884, + "pictured": 15885, + "distract": 15886, + "fixing": 15887, + "fiddle": 15888, + "giggled": 15889, + "burger": 15890, + "heirs": 15891, + "hairy": 15892, + "mechanic": 15893, + "torque": 15894, + "apache": 15895, + "obsessed": 15896, + "chiefly": 15897, + "cheng": 15898, + "logging": 15899, + "##tag": 15900, + "extracted": 15901, + "meaningful": 15902, + "numb": 15903, + "##vsky": 15904, + "gloucestershire": 15905, + "reminding": 15906, + "##bay": 15907, + "unite": 15908, + "##lit": 15909, + "breeds": 15910, + "diminished": 15911, + "clown": 15912, + "glove": 15913, + "1860s": 15914, + "##ن": 15915, + "##ug": 15916, + "archibald": 15917, + "focal": 15918, + "freelance": 15919, + "sliced": 15920, + "depiction": 15921, + "##yk": 15922, + "organism": 15923, + "switches": 15924, + "sights": 15925, + "stray": 15926, + "crawling": 15927, + "##ril": 15928, + "lever": 15929, + "leningrad": 15930, + "interpretations": 15931, + "loops": 15932, + "anytime": 15933, + "reel": 15934, + "alicia": 15935, + "delighted": 15936, + "##ech": 15937, + "inhaled": 15938, + "xiv": 15939, + "suitcase": 15940, + "bernie": 15941, + "vega": 15942, + "licenses": 15943, + "northampton": 15944, + "exclusion": 15945, + "induction": 15946, + "monasteries": 15947, + "racecourse": 15948, + "homosexuality": 15949, + "##right": 15950, + "##sfield": 15951, + "##rky": 15952, + "dimitri": 15953, + "michele": 15954, + "alternatives": 15955, + "ions": 15956, + "commentators": 15957, + "genuinely": 15958, + "objected": 15959, + "pork": 15960, + "hospitality": 15961, + "fencing": 15962, + "stephan": 15963, + "warships": 15964, + "peripheral": 15965, + "wit": 15966, + "drunken": 15967, + "wrinkled": 15968, + "quentin": 15969, + "spends": 15970, + "departing": 15971, + "chung": 15972, + "numerical": 15973, + "spokesperson": 15974, + "##zone": 15975, + "johannesburg": 15976, + "caliber": 15977, + "killers": 15978, + "##udge": 15979, + "assumes": 15980, + "neatly": 15981, + "demographic": 15982, + "abigail": 15983, + "bloc": 15984, + "##vel": 15985, + "mounting": 15986, + "##lain": 15987, + "bentley": 15988, + "slightest": 15989, + "xu": 15990, + "recipients": 15991, + "##jk": 15992, + "merlin": 15993, + "##writer": 15994, + "seniors": 15995, + "prisons": 15996, + "blinking": 15997, + "hindwings": 15998, + "flickered": 15999, + "kappa": 16000, + "##hel": 16001, + "80s": 16002, + "strengthening": 16003, + "appealing": 16004, + "brewing": 16005, + "gypsy": 16006, + "mali": 16007, + "lashes": 16008, + "hulk": 16009, + "unpleasant": 16010, + "harassment": 16011, + "bio": 16012, + "treaties": 16013, + "predict": 16014, + "instrumentation": 16015, + "pulp": 16016, + "troupe": 16017, + "boiling": 16018, + "mantle": 16019, + "##ffe": 16020, + "ins": 16021, + "##vn": 16022, + "dividing": 16023, + "handles": 16024, + "verbs": 16025, + "##onal": 16026, + "coconut": 16027, + "senegal": 16028, + "340": 16029, + "thorough": 16030, + "gum": 16031, + "momentarily": 16032, + "##sto": 16033, + "cocaine": 16034, + "panicked": 16035, + "destined": 16036, + "##turing": 16037, + "teatro": 16038, + "denying": 16039, + "weary": 16040, + "captained": 16041, + "mans": 16042, + "##hawks": 16043, + "##code": 16044, + "wakefield": 16045, + "bollywood": 16046, + "thankfully": 16047, + "##16": 16048, + "cyril": 16049, + "##wu": 16050, + "amendments": 16051, + "##bahn": 16052, + "consultation": 16053, + "stud": 16054, + "reflections": 16055, + "kindness": 16056, + "1787": 16057, + "internally": 16058, + "##ovo": 16059, + "tex": 16060, + "mosaic": 16061, + "distribute": 16062, + "paddy": 16063, + "seeming": 16064, + "143": 16065, + "##hic": 16066, + "piers": 16067, + "##15": 16068, + "##mura": 16069, + "##verse": 16070, + "popularly": 16071, + "winger": 16072, + "kang": 16073, + "sentinel": 16074, + "mccoy": 16075, + "##anza": 16076, + "covenant": 16077, + "##bag": 16078, + "verge": 16079, + "fireworks": 16080, + "suppress": 16081, + "thrilled": 16082, + "dominate": 16083, + "##jar": 16084, + "swansea": 16085, + "##60": 16086, + "142": 16087, + "reconciliation": 16088, + "##ndi": 16089, + "stiffened": 16090, + "cue": 16091, + "dorian": 16092, + "##uf": 16093, + "damascus": 16094, + "amor": 16095, + "ida": 16096, + "foremost": 16097, + "##aga": 16098, + "porsche": 16099, + "unseen": 16100, + "dir": 16101, + "##had": 16102, + "##azi": 16103, + "stony": 16104, + "lexi": 16105, + "melodies": 16106, + "##nko": 16107, + "angular": 16108, + "integer": 16109, + "podcast": 16110, + "ants": 16111, + "inherent": 16112, + "jaws": 16113, + "justify": 16114, + "persona": 16115, + "##olved": 16116, + "josephine": 16117, + "##nr": 16118, + "##ressed": 16119, + "customary": 16120, + "flashes": 16121, + "gala": 16122, + "cyrus": 16123, + "glaring": 16124, + "backyard": 16125, + "ariel": 16126, + "physiology": 16127, + "greenland": 16128, + "html": 16129, + "stir": 16130, + "avon": 16131, + "atletico": 16132, + "finch": 16133, + "methodology": 16134, + "ked": 16135, + "##lent": 16136, + "mas": 16137, + "catholicism": 16138, + "townsend": 16139, + "branding": 16140, + "quincy": 16141, + "fits": 16142, + "containers": 16143, + "1777": 16144, + "ashore": 16145, + "aragon": 16146, + "##19": 16147, + "forearm": 16148, + "poisoning": 16149, + "##sd": 16150, + "adopting": 16151, + "conquer": 16152, + "grinding": 16153, + "amnesty": 16154, + "keller": 16155, + "finances": 16156, + "evaluate": 16157, + "forged": 16158, + "lankan": 16159, + "instincts": 16160, + "##uto": 16161, + "guam": 16162, + "bosnian": 16163, + "photographed": 16164, + "workplace": 16165, + "desirable": 16166, + "protector": 16167, + "##dog": 16168, + "allocation": 16169, + "intently": 16170, + "encourages": 16171, + "willy": 16172, + "##sten": 16173, + "bodyguard": 16174, + "electro": 16175, + "brighter": 16176, + "##ν": 16177, + "bihar": 16178, + "##chev": 16179, + "lasts": 16180, + "opener": 16181, + "amphibious": 16182, + "sal": 16183, + "verde": 16184, + "arte": 16185, + "##cope": 16186, + "captivity": 16187, + "vocabulary": 16188, + "yields": 16189, + "##tted": 16190, + "agreeing": 16191, + "desmond": 16192, + "pioneered": 16193, + "##chus": 16194, + "strap": 16195, + "campaigned": 16196, + "railroads": 16197, + "##ович": 16198, + "emblem": 16199, + "##dre": 16200, + "stormed": 16201, + "501": 16202, + "##ulous": 16203, + "marijuana": 16204, + "northumberland": 16205, + "##gn": 16206, + "##nath": 16207, + "bowen": 16208, + "landmarks": 16209, + "beaumont": 16210, + "##qua": 16211, + "danube": 16212, + "##bler": 16213, + "attorneys": 16214, + "th": 16215, + "ge": 16216, + "flyers": 16217, + "critique": 16218, + "villains": 16219, + "cass": 16220, + "mutation": 16221, + "acc": 16222, + "##0s": 16223, + "colombo": 16224, + "mckay": 16225, + "motif": 16226, + "sampling": 16227, + "concluding": 16228, + "syndicate": 16229, + "##rell": 16230, + "neon": 16231, + "stables": 16232, + "ds": 16233, + "warnings": 16234, + "clint": 16235, + "mourning": 16236, + "wilkinson": 16237, + "##tated": 16238, + "merrill": 16239, + "leopard": 16240, + "evenings": 16241, + "exhaled": 16242, + "emil": 16243, + "sonia": 16244, + "ezra": 16245, + "discrete": 16246, + "stove": 16247, + "farrell": 16248, + "fifteenth": 16249, + "prescribed": 16250, + "superhero": 16251, + "##rier": 16252, + "worms": 16253, + "helm": 16254, + "wren": 16255, + "##duction": 16256, + "##hc": 16257, + "expo": 16258, + "##rator": 16259, + "hq": 16260, + "unfamiliar": 16261, + "antony": 16262, + "prevents": 16263, + "acceleration": 16264, + "fiercely": 16265, + "mari": 16266, + "painfully": 16267, + "calculations": 16268, + "cheaper": 16269, + "ign": 16270, + "clifton": 16271, + "irvine": 16272, + "davenport": 16273, + "mozambique": 16274, + "##np": 16275, + "pierced": 16276, + "##evich": 16277, + "wonders": 16278, + "##wig": 16279, + "##cate": 16280, + "##iling": 16281, + "crusade": 16282, + "ware": 16283, + "##uel": 16284, + "enzymes": 16285, + "reasonably": 16286, + "mls": 16287, + "##coe": 16288, + "mater": 16289, + "ambition": 16290, + "bunny": 16291, + "eliot": 16292, + "kernel": 16293, + "##fin": 16294, + "asphalt": 16295, + "headmaster": 16296, + "torah": 16297, + "aden": 16298, + "lush": 16299, + "pins": 16300, + "waived": 16301, + "##care": 16302, + "##yas": 16303, + "joao": 16304, + "substrate": 16305, + "enforce": 16306, + "##grad": 16307, + "##ules": 16308, + "alvarez": 16309, + "selections": 16310, + "epidemic": 16311, + "tempted": 16312, + "##bit": 16313, + "bremen": 16314, + "translates": 16315, + "ensured": 16316, + "waterfront": 16317, + "29th": 16318, + "forrest": 16319, + "manny": 16320, + "malone": 16321, + "kramer": 16322, + "reigning": 16323, + "cookies": 16324, + "simpler": 16325, + "absorption": 16326, + "205": 16327, + "engraved": 16328, + "##ffy": 16329, + "evaluated": 16330, + "1778": 16331, + "haze": 16332, + "146": 16333, + "comforting": 16334, + "crossover": 16335, + "##abe": 16336, + "thorn": 16337, + "##rift": 16338, + "##imo": 16339, + "##pop": 16340, + "suppression": 16341, + "fatigue": 16342, + "cutter": 16343, + "##tr": 16344, + "201": 16345, + "wurttemberg": 16346, + "##orf": 16347, + "enforced": 16348, + "hovering": 16349, + "proprietary": 16350, + "gb": 16351, + "samurai": 16352, + "syllable": 16353, + "ascent": 16354, + "lacey": 16355, + "tick": 16356, + "lars": 16357, + "tractor": 16358, + "merchandise": 16359, + "rep": 16360, + "bouncing": 16361, + "defendants": 16362, + "##yre": 16363, + "huntington": 16364, + "##ground": 16365, + "##oko": 16366, + "standardized": 16367, + "##hor": 16368, + "##hima": 16369, + "assassinated": 16370, + "nu": 16371, + "predecessors": 16372, + "rainy": 16373, + "liar": 16374, + "assurance": 16375, + "lyrical": 16376, + "##uga": 16377, + "secondly": 16378, + "flattened": 16379, + "ios": 16380, + "parameter": 16381, + "undercover": 16382, + "##mity": 16383, + "bordeaux": 16384, + "punish": 16385, + "ridges": 16386, + "markers": 16387, + "exodus": 16388, + "inactive": 16389, + "hesitate": 16390, + "debbie": 16391, + "nyc": 16392, + "pledge": 16393, + "savoy": 16394, + "nagar": 16395, + "offset": 16396, + "organist": 16397, + "##tium": 16398, + "hesse": 16399, + "marin": 16400, + "converting": 16401, + "##iver": 16402, + "diagram": 16403, + "propulsion": 16404, + "pu": 16405, + "validity": 16406, + "reverted": 16407, + "supportive": 16408, + "##dc": 16409, + "ministries": 16410, + "clans": 16411, + "responds": 16412, + "proclamation": 16413, + "##inae": 16414, + "##ø": 16415, + "##rea": 16416, + "ein": 16417, + "pleading": 16418, + "patriot": 16419, + "sf": 16420, + "birch": 16421, + "islanders": 16422, + "strauss": 16423, + "hates": 16424, + "##dh": 16425, + "brandenburg": 16426, + "concession": 16427, + "rd": 16428, + "##ob": 16429, + "1900s": 16430, + "killings": 16431, + "textbook": 16432, + "antiquity": 16433, + "cinematography": 16434, + "wharf": 16435, + "embarrassing": 16436, + "setup": 16437, + "creed": 16438, + "farmland": 16439, + "inequality": 16440, + "centred": 16441, + "signatures": 16442, + "fallon": 16443, + "370": 16444, + "##ingham": 16445, + "##uts": 16446, + "ceylon": 16447, + "gazing": 16448, + "directive": 16449, + "laurie": 16450, + "##tern": 16451, + "globally": 16452, + "##uated": 16453, + "##dent": 16454, + "allah": 16455, + "excavation": 16456, + "threads": 16457, + "##cross": 16458, + "148": 16459, + "frantically": 16460, + "icc": 16461, + "utilize": 16462, + "determines": 16463, + "respiratory": 16464, + "thoughtful": 16465, + "receptions": 16466, + "##dicate": 16467, + "merging": 16468, + "chandra": 16469, + "seine": 16470, + "147": 16471, + "builders": 16472, + "builds": 16473, + "diagnostic": 16474, + "dev": 16475, + "visibility": 16476, + "goddamn": 16477, + "analyses": 16478, + "dhaka": 16479, + "cho": 16480, + "proves": 16481, + "chancel": 16482, + "concurrent": 16483, + "curiously": 16484, + "canadians": 16485, + "pumped": 16486, + "restoring": 16487, + "1850s": 16488, + "turtles": 16489, + "jaguar": 16490, + "sinister": 16491, + "spinal": 16492, + "traction": 16493, + "declan": 16494, + "vows": 16495, + "1784": 16496, + "glowed": 16497, + "capitalism": 16498, + "swirling": 16499, + "install": 16500, + "universidad": 16501, + "##lder": 16502, + "##oat": 16503, + "soloist": 16504, + "##genic": 16505, + "##oor": 16506, + "coincidence": 16507, + "beginnings": 16508, + "nissan": 16509, + "dip": 16510, + "resorts": 16511, + "caucasus": 16512, + "combustion": 16513, + "infectious": 16514, + "##eno": 16515, + "pigeon": 16516, + "serpent": 16517, + "##itating": 16518, + "conclude": 16519, + "masked": 16520, + "salad": 16521, + "jew": 16522, + "##gr": 16523, + "surreal": 16524, + "toni": 16525, + "##wc": 16526, + "harmonica": 16527, + "151": 16528, + "##gins": 16529, + "##etic": 16530, + "##coat": 16531, + "fishermen": 16532, + "intending": 16533, + "bravery": 16534, + "##wave": 16535, + "klaus": 16536, + "titan": 16537, + "wembley": 16538, + "taiwanese": 16539, + "ransom": 16540, + "40th": 16541, + "incorrect": 16542, + "hussein": 16543, + "eyelids": 16544, + "jp": 16545, + "cooke": 16546, + "dramas": 16547, + "utilities": 16548, + "##etta": 16549, + "##print": 16550, + "eisenhower": 16551, + "principally": 16552, + "granada": 16553, + "lana": 16554, + "##rak": 16555, + "openings": 16556, + "concord": 16557, + "##bl": 16558, + "bethany": 16559, + "connie": 16560, + "morality": 16561, + "sega": 16562, + "##mons": 16563, + "##nard": 16564, + "earnings": 16565, + "##kara": 16566, + "##cine": 16567, + "wii": 16568, + "communes": 16569, + "##rel": 16570, + "coma": 16571, + "composing": 16572, + "softened": 16573, + "severed": 16574, + "grapes": 16575, + "##17": 16576, + "nguyen": 16577, + "analyzed": 16578, + "warlord": 16579, + "hubbard": 16580, + "heavenly": 16581, + "behave": 16582, + "slovenian": 16583, + "##hit": 16584, + "##ony": 16585, + "hailed": 16586, + "filmmakers": 16587, + "trance": 16588, + "caldwell": 16589, + "skye": 16590, + "unrest": 16591, + "coward": 16592, + "likelihood": 16593, + "##aging": 16594, + "bern": 16595, + "sci": 16596, + "taliban": 16597, + "honolulu": 16598, + "propose": 16599, + "##wang": 16600, + "1700": 16601, + "browser": 16602, + "imagining": 16603, + "cobra": 16604, + "contributes": 16605, + "dukes": 16606, + "instinctively": 16607, + "conan": 16608, + "violinist": 16609, + "##ores": 16610, + "accessories": 16611, + "gradual": 16612, + "##amp": 16613, + "quotes": 16614, + "sioux": 16615, + "##dating": 16616, + "undertake": 16617, + "intercepted": 16618, + "sparkling": 16619, + "compressed": 16620, + "139": 16621, + "fungus": 16622, + "tombs": 16623, + "haley": 16624, + "imposing": 16625, + "rests": 16626, + "degradation": 16627, + "lincolnshire": 16628, + "retailers": 16629, + "wetlands": 16630, + "tulsa": 16631, + "distributor": 16632, + "dungeon": 16633, + "nun": 16634, + "greenhouse": 16635, + "convey": 16636, + "atlantis": 16637, + "aft": 16638, + "exits": 16639, + "oman": 16640, + "dresser": 16641, + "lyons": 16642, + "##sti": 16643, + "joking": 16644, + "eddy": 16645, + "judgement": 16646, + "omitted": 16647, + "digits": 16648, + "##cts": 16649, + "##game": 16650, + "juniors": 16651, + "##rae": 16652, + "cents": 16653, + "stricken": 16654, + "une": 16655, + "##ngo": 16656, + "wizards": 16657, + "weir": 16658, + "breton": 16659, + "nan": 16660, + "technician": 16661, + "fibers": 16662, + "liking": 16663, + "royalty": 16664, + "##cca": 16665, + "154": 16666, + "persia": 16667, + "terribly": 16668, + "magician": 16669, + "##rable": 16670, + "##unt": 16671, + "vance": 16672, + "cafeteria": 16673, + "booker": 16674, + "camille": 16675, + "warmer": 16676, + "##static": 16677, + "consume": 16678, + "cavern": 16679, + "gaps": 16680, + "compass": 16681, + "contemporaries": 16682, + "foyer": 16683, + "soothing": 16684, + "graveyard": 16685, + "maj": 16686, + "plunged": 16687, + "blush": 16688, + "##wear": 16689, + "cascade": 16690, + "demonstrates": 16691, + "ordinance": 16692, + "##nov": 16693, + "boyle": 16694, + "##lana": 16695, + "rockefeller": 16696, + "shaken": 16697, + "banjo": 16698, + "izzy": 16699, + "##ense": 16700, + "breathless": 16701, + "vines": 16702, + "##32": 16703, + "##eman": 16704, + "alterations": 16705, + "chromosome": 16706, + "dwellings": 16707, + "feudal": 16708, + "mole": 16709, + "153": 16710, + "catalonia": 16711, + "relics": 16712, + "tenant": 16713, + "mandated": 16714, + "##fm": 16715, + "fridge": 16716, + "hats": 16717, + "honesty": 16718, + "patented": 16719, + "raul": 16720, + "heap": 16721, + "cruisers": 16722, + "accusing": 16723, + "enlightenment": 16724, + "infants": 16725, + "wherein": 16726, + "chatham": 16727, + "contractors": 16728, + "zen": 16729, + "affinity": 16730, + "hc": 16731, + "osborne": 16732, + "piston": 16733, + "156": 16734, + "traps": 16735, + "maturity": 16736, + "##rana": 16737, + "lagos": 16738, + "##zal": 16739, + "peering": 16740, + "##nay": 16741, + "attendant": 16742, + "dealers": 16743, + "protocols": 16744, + "subset": 16745, + "prospects": 16746, + "biographical": 16747, + "##cre": 16748, + "artery": 16749, + "##zers": 16750, + "insignia": 16751, + "nuns": 16752, + "endured": 16753, + "##eration": 16754, + "recommend": 16755, + "schwartz": 16756, + "serbs": 16757, + "berger": 16758, + "cromwell": 16759, + "crossroads": 16760, + "##ctor": 16761, + "enduring": 16762, + "clasped": 16763, + "grounded": 16764, + "##bine": 16765, + "marseille": 16766, + "twitched": 16767, + "abel": 16768, + "choke": 16769, + "https": 16770, + "catalyst": 16771, + "moldova": 16772, + "italians": 16773, + "##tist": 16774, + "disastrous": 16775, + "wee": 16776, + "##oured": 16777, + "##nti": 16778, + "wwf": 16779, + "nope": 16780, + "##piration": 16781, + "##asa": 16782, + "expresses": 16783, + "thumbs": 16784, + "167": 16785, + "##nza": 16786, + "coca": 16787, + "1781": 16788, + "cheating": 16789, + "##ption": 16790, + "skipped": 16791, + "sensory": 16792, + "heidelberg": 16793, + "spies": 16794, + "satan": 16795, + "dangers": 16796, + "semifinal": 16797, + "202": 16798, + "bohemia": 16799, + "whitish": 16800, + "confusing": 16801, + "shipbuilding": 16802, + "relies": 16803, + "surgeons": 16804, + "landings": 16805, + "ravi": 16806, + "baku": 16807, + "moor": 16808, + "suffix": 16809, + "alejandro": 16810, + "##yana": 16811, + "litre": 16812, + "upheld": 16813, + "##unk": 16814, + "rajasthan": 16815, + "##rek": 16816, + "coaster": 16817, + "insists": 16818, + "posture": 16819, + "scenarios": 16820, + "etienne": 16821, + "favoured": 16822, + "appoint": 16823, + "transgender": 16824, + "elephants": 16825, + "poked": 16826, + "greenwood": 16827, + "defences": 16828, + "fulfilled": 16829, + "militant": 16830, + "somali": 16831, + "1758": 16832, + "chalk": 16833, + "potent": 16834, + "##ucci": 16835, + "migrants": 16836, + "wink": 16837, + "assistants": 16838, + "nos": 16839, + "restriction": 16840, + "activism": 16841, + "niger": 16842, + "##ario": 16843, + "colon": 16844, + "shaun": 16845, + "##sat": 16846, + "daphne": 16847, + "##erated": 16848, + "swam": 16849, + "congregations": 16850, + "reprise": 16851, + "considerations": 16852, + "magnet": 16853, + "playable": 16854, + "xvi": 16855, + "##р": 16856, + "overthrow": 16857, + "tobias": 16858, + "knob": 16859, + "chavez": 16860, + "coding": 16861, + "##mers": 16862, + "propped": 16863, + "katrina": 16864, + "orient": 16865, + "newcomer": 16866, + "##suke": 16867, + "temperate": 16868, + "##pool": 16869, + "farmhouse": 16870, + "interrogation": 16871, + "##vd": 16872, + "committing": 16873, + "##vert": 16874, + "forthcoming": 16875, + "strawberry": 16876, + "joaquin": 16877, + "macau": 16878, + "ponds": 16879, + "shocking": 16880, + "siberia": 16881, + "##cellular": 16882, + "chant": 16883, + "contributors": 16884, + "##nant": 16885, + "##ologists": 16886, + "sped": 16887, + "absorb": 16888, + "hail": 16889, + "1782": 16890, + "spared": 16891, + "##hore": 16892, + "barbados": 16893, + "karate": 16894, + "opus": 16895, + "originates": 16896, + "saul": 16897, + "##xie": 16898, + "evergreen": 16899, + "leaped": 16900, + "##rock": 16901, + "correlation": 16902, + "exaggerated": 16903, + "weekday": 16904, + "unification": 16905, + "bump": 16906, + "tracing": 16907, + "brig": 16908, + "afb": 16909, + "pathways": 16910, + "utilizing": 16911, + "##ners": 16912, + "mod": 16913, + "mb": 16914, + "disturbance": 16915, + "kneeling": 16916, + "##stad": 16917, + "##guchi": 16918, + "100th": 16919, + "pune": 16920, + "##thy": 16921, + "decreasing": 16922, + "168": 16923, + "manipulation": 16924, + "miriam": 16925, + "academia": 16926, + "ecosystem": 16927, + "occupational": 16928, + "rbi": 16929, + "##lem": 16930, + "rift": 16931, + "##14": 16932, + "rotary": 16933, + "stacked": 16934, + "incorporation": 16935, + "awakening": 16936, + "generators": 16937, + "guerrero": 16938, + "racist": 16939, + "##omy": 16940, + "cyber": 16941, + "derivatives": 16942, + "culminated": 16943, + "allie": 16944, + "annals": 16945, + "panzer": 16946, + "sainte": 16947, + "wikipedia": 16948, + "pops": 16949, + "zu": 16950, + "austro": 16951, + "##vate": 16952, + "algerian": 16953, + "politely": 16954, + "nicholson": 16955, + "mornings": 16956, + "educate": 16957, + "tastes": 16958, + "thrill": 16959, + "dartmouth": 16960, + "##gating": 16961, + "db": 16962, + "##jee": 16963, + "regan": 16964, + "differing": 16965, + "concentrating": 16966, + "choreography": 16967, + "divinity": 16968, + "##media": 16969, + "pledged": 16970, + "alexandre": 16971, + "routing": 16972, + "gregor": 16973, + "madeline": 16974, + "##idal": 16975, + "apocalypse": 16976, + "##hora": 16977, + "gunfire": 16978, + "culminating": 16979, + "elves": 16980, + "fined": 16981, + "liang": 16982, + "lam": 16983, + "programmed": 16984, + "tar": 16985, + "guessing": 16986, + "transparency": 16987, + "gabrielle": 16988, + "##gna": 16989, + "cancellation": 16990, + "flexibility": 16991, + "##lining": 16992, + "accession": 16993, + "shea": 16994, + "stronghold": 16995, + "nets": 16996, + "specializes": 16997, + "##rgan": 16998, + "abused": 16999, + "hasan": 17000, + "sgt": 17001, + "ling": 17002, + "exceeding": 17003, + "##₄": 17004, + "admiration": 17005, + "supermarket": 17006, + "##ark": 17007, + "photographers": 17008, + "specialised": 17009, + "tilt": 17010, + "resonance": 17011, + "hmm": 17012, + "perfume": 17013, + "380": 17014, + "sami": 17015, + "threatens": 17016, + "garland": 17017, + "botany": 17018, + "guarding": 17019, + "boiled": 17020, + "greet": 17021, + "puppy": 17022, + "russo": 17023, + "supplier": 17024, + "wilmington": 17025, + "vibrant": 17026, + "vijay": 17027, + "##bius": 17028, + "paralympic": 17029, + "grumbled": 17030, + "paige": 17031, + "faa": 17032, + "licking": 17033, + "margins": 17034, + "hurricanes": 17035, + "##gong": 17036, + "fest": 17037, + "grenade": 17038, + "ripping": 17039, + "##uz": 17040, + "counseling": 17041, + "weigh": 17042, + "##sian": 17043, + "needles": 17044, + "wiltshire": 17045, + "edison": 17046, + "costly": 17047, + "##not": 17048, + "fulton": 17049, + "tramway": 17050, + "redesigned": 17051, + "staffordshire": 17052, + "cache": 17053, + "gasping": 17054, + "watkins": 17055, + "sleepy": 17056, + "candidacy": 17057, + "##group": 17058, + "monkeys": 17059, + "timeline": 17060, + "throbbing": 17061, + "##bid": 17062, + "##sos": 17063, + "berth": 17064, + "uzbekistan": 17065, + "vanderbilt": 17066, + "bothering": 17067, + "overturned": 17068, + "ballots": 17069, + "gem": 17070, + "##iger": 17071, + "sunglasses": 17072, + "subscribers": 17073, + "hooker": 17074, + "compelling": 17075, + "ang": 17076, + "exceptionally": 17077, + "saloon": 17078, + "stab": 17079, + "##rdi": 17080, + "carla": 17081, + "terrifying": 17082, + "rom": 17083, + "##vision": 17084, + "coil": 17085, + "##oids": 17086, + "satisfying": 17087, + "vendors": 17088, + "31st": 17089, + "mackay": 17090, + "deities": 17091, + "overlooked": 17092, + "ambient": 17093, + "bahamas": 17094, + "felipe": 17095, + "olympia": 17096, + "whirled": 17097, + "botanist": 17098, + "advertised": 17099, + "tugging": 17100, + "##dden": 17101, + "disciples": 17102, + "morales": 17103, + "unionist": 17104, + "rites": 17105, + "foley": 17106, + "morse": 17107, + "motives": 17108, + "creepy": 17109, + "##₀": 17110, + "soo": 17111, + "##sz": 17112, + "bargain": 17113, + "highness": 17114, + "frightening": 17115, + "turnpike": 17116, + "tory": 17117, + "reorganization": 17118, + "##cer": 17119, + "depict": 17120, + "biographer": 17121, + "##walk": 17122, + "unopposed": 17123, + "manifesto": 17124, + "##gles": 17125, + "institut": 17126, + "emile": 17127, + "accidental": 17128, + "kapoor": 17129, + "##dam": 17130, + "kilkenny": 17131, + "cortex": 17132, + "lively": 17133, + "##13": 17134, + "romanesque": 17135, + "jain": 17136, + "shan": 17137, + "cannons": 17138, + "##ood": 17139, + "##ske": 17140, + "petrol": 17141, + "echoing": 17142, + "amalgamated": 17143, + "disappears": 17144, + "cautious": 17145, + "proposes": 17146, + "sanctions": 17147, + "trenton": 17148, + "##ر": 17149, + "flotilla": 17150, + "aus": 17151, + "contempt": 17152, + "tor": 17153, + "canary": 17154, + "cote": 17155, + "theirs": 17156, + "##hun": 17157, + "conceptual": 17158, + "deleted": 17159, + "fascinating": 17160, + "paso": 17161, + "blazing": 17162, + "elf": 17163, + "honourable": 17164, + "hutchinson": 17165, + "##eiro": 17166, + "##outh": 17167, + "##zin": 17168, + "surveyor": 17169, + "tee": 17170, + "amidst": 17171, + "wooded": 17172, + "reissue": 17173, + "intro": 17174, + "##ono": 17175, + "cobb": 17176, + "shelters": 17177, + "newsletter": 17178, + "hanson": 17179, + "brace": 17180, + "encoding": 17181, + "confiscated": 17182, + "dem": 17183, + "caravan": 17184, + "marino": 17185, + "scroll": 17186, + "melodic": 17187, + "cows": 17188, + "imam": 17189, + "##adi": 17190, + "##aneous": 17191, + "northward": 17192, + "searches": 17193, + "biodiversity": 17194, + "cora": 17195, + "310": 17196, + "roaring": 17197, + "##bers": 17198, + "connell": 17199, + "theologian": 17200, + "halo": 17201, + "compose": 17202, + "pathetic": 17203, + "unmarried": 17204, + "dynamo": 17205, + "##oot": 17206, + "az": 17207, + "calculation": 17208, + "toulouse": 17209, + "deserves": 17210, + "humour": 17211, + "nr": 17212, + "forgiveness": 17213, + "tam": 17214, + "undergone": 17215, + "martyr": 17216, + "pamela": 17217, + "myths": 17218, + "whore": 17219, + "counselor": 17220, + "hicks": 17221, + "290": 17222, + "heavens": 17223, + "battleship": 17224, + "electromagnetic": 17225, + "##bbs": 17226, + "stellar": 17227, + "establishments": 17228, + "presley": 17229, + "hopped": 17230, + "##chin": 17231, + "temptation": 17232, + "90s": 17233, + "wills": 17234, + "nas": 17235, + "##yuan": 17236, + "nhs": 17237, + "##nya": 17238, + "seminars": 17239, + "##yev": 17240, + "adaptations": 17241, + "gong": 17242, + "asher": 17243, + "lex": 17244, + "indicator": 17245, + "sikh": 17246, + "tobago": 17247, + "cites": 17248, + "goin": 17249, + "##yte": 17250, + "satirical": 17251, + "##gies": 17252, + "characterised": 17253, + "correspond": 17254, + "bubbles": 17255, + "lure": 17256, + "participates": 17257, + "##vid": 17258, + "eruption": 17259, + "skate": 17260, + "therapeutic": 17261, + "1785": 17262, + "canals": 17263, + "wholesale": 17264, + "defaulted": 17265, + "sac": 17266, + "460": 17267, + "petit": 17268, + "##zzled": 17269, + "virgil": 17270, + "leak": 17271, + "ravens": 17272, + "256": 17273, + "portraying": 17274, + "##yx": 17275, + "ghetto": 17276, + "creators": 17277, + "dams": 17278, + "portray": 17279, + "vicente": 17280, + "##rington": 17281, + "fae": 17282, + "namesake": 17283, + "bounty": 17284, + "##arium": 17285, + "joachim": 17286, + "##ota": 17287, + "##iser": 17288, + "aforementioned": 17289, + "axle": 17290, + "snout": 17291, + "depended": 17292, + "dismantled": 17293, + "reuben": 17294, + "480": 17295, + "##ibly": 17296, + "gallagher": 17297, + "##lau": 17298, + "##pd": 17299, + "earnest": 17300, + "##ieu": 17301, + "##iary": 17302, + "inflicted": 17303, + "objections": 17304, + "##llar": 17305, + "asa": 17306, + "gritted": 17307, + "##athy": 17308, + "jericho": 17309, + "##sea": 17310, + "##was": 17311, + "flick": 17312, + "underside": 17313, + "ceramics": 17314, + "undead": 17315, + "substituted": 17316, + "195": 17317, + "eastward": 17318, + "undoubtedly": 17319, + "wheeled": 17320, + "chimney": 17321, + "##iche": 17322, + "guinness": 17323, + "cb": 17324, + "##ager": 17325, + "siding": 17326, + "##bell": 17327, + "traitor": 17328, + "baptiste": 17329, + "disguised": 17330, + "inauguration": 17331, + "149": 17332, + "tipperary": 17333, + "choreographer": 17334, + "perched": 17335, + "warmed": 17336, + "stationary": 17337, + "eco": 17338, + "##ike": 17339, + "##ntes": 17340, + "bacterial": 17341, + "##aurus": 17342, + "flores": 17343, + "phosphate": 17344, + "##core": 17345, + "attacker": 17346, + "invaders": 17347, + "alvin": 17348, + "intersects": 17349, + "a1": 17350, + "indirectly": 17351, + "immigrated": 17352, + "businessmen": 17353, + "cornelius": 17354, + "valves": 17355, + "narrated": 17356, + "pill": 17357, + "sober": 17358, + "ul": 17359, + "nationale": 17360, + "monastic": 17361, + "applicants": 17362, + "scenery": 17363, + "##jack": 17364, + "161": 17365, + "motifs": 17366, + "constitutes": 17367, + "cpu": 17368, + "##osh": 17369, + "jurisdictions": 17370, + "sd": 17371, + "tuning": 17372, + "irritation": 17373, + "woven": 17374, + "##uddin": 17375, + "fertility": 17376, + "gao": 17377, + "##erie": 17378, + "antagonist": 17379, + "impatient": 17380, + "glacial": 17381, + "hides": 17382, + "boarded": 17383, + "denominations": 17384, + "interception": 17385, + "##jas": 17386, + "cookie": 17387, + "nicola": 17388, + "##tee": 17389, + "algebraic": 17390, + "marquess": 17391, + "bahn": 17392, + "parole": 17393, + "buyers": 17394, + "bait": 17395, + "turbines": 17396, + "paperwork": 17397, + "bestowed": 17398, + "natasha": 17399, + "renee": 17400, + "oceans": 17401, + "purchases": 17402, + "157": 17403, + "vaccine": 17404, + "215": 17405, + "##tock": 17406, + "fixtures": 17407, + "playhouse": 17408, + "integrate": 17409, + "jai": 17410, + "oswald": 17411, + "intellectuals": 17412, + "##cky": 17413, + "booked": 17414, + "nests": 17415, + "mortimer": 17416, + "##isi": 17417, + "obsession": 17418, + "sept": 17419, + "##gler": 17420, + "##sum": 17421, + "440": 17422, + "scrutiny": 17423, + "simultaneous": 17424, + "squinted": 17425, + "##shin": 17426, + "collects": 17427, + "oven": 17428, + "shankar": 17429, + "penned": 17430, + "remarkably": 17431, + "##я": 17432, + "slips": 17433, + "luggage": 17434, + "spectral": 17435, + "1786": 17436, + "collaborations": 17437, + "louie": 17438, + "consolidation": 17439, + "##ailed": 17440, + "##ivating": 17441, + "420": 17442, + "hoover": 17443, + "blackpool": 17444, + "harness": 17445, + "ignition": 17446, + "vest": 17447, + "tails": 17448, + "belmont": 17449, + "mongol": 17450, + "skinner": 17451, + "##nae": 17452, + "visually": 17453, + "mage": 17454, + "derry": 17455, + "##tism": 17456, + "##unce": 17457, + "stevie": 17458, + "transitional": 17459, + "##rdy": 17460, + "redskins": 17461, + "drying": 17462, + "prep": 17463, + "prospective": 17464, + "##21": 17465, + "annoyance": 17466, + "oversee": 17467, + "##loaded": 17468, + "fills": 17469, + "##books": 17470, + "##iki": 17471, + "announces": 17472, + "fda": 17473, + "scowled": 17474, + "respects": 17475, + "prasad": 17476, + "mystic": 17477, + "tucson": 17478, + "##vale": 17479, + "revue": 17480, + "springer": 17481, + "bankrupt": 17482, + "1772": 17483, + "aristotle": 17484, + "salvatore": 17485, + "habsburg": 17486, + "##geny": 17487, + "dal": 17488, + "natal": 17489, + "nut": 17490, + "pod": 17491, + "chewing": 17492, + "darts": 17493, + "moroccan": 17494, + "walkover": 17495, + "rosario": 17496, + "lenin": 17497, + "punjabi": 17498, + "##ße": 17499, + "grossed": 17500, + "scattering": 17501, + "wired": 17502, + "invasive": 17503, + "hui": 17504, + "polynomial": 17505, + "corridors": 17506, + "wakes": 17507, + "gina": 17508, + "portrays": 17509, + "##cratic": 17510, + "arid": 17511, + "retreating": 17512, + "erich": 17513, + "irwin": 17514, + "sniper": 17515, + "##dha": 17516, + "linen": 17517, + "lindsey": 17518, + "maneuver": 17519, + "butch": 17520, + "shutting": 17521, + "socio": 17522, + "bounce": 17523, + "commemorative": 17524, + "postseason": 17525, + "jeremiah": 17526, + "pines": 17527, + "275": 17528, + "mystical": 17529, + "beads": 17530, + "bp": 17531, + "abbas": 17532, + "furnace": 17533, + "bidding": 17534, + "consulted": 17535, + "assaulted": 17536, + "empirical": 17537, + "rubble": 17538, + "enclosure": 17539, + "sob": 17540, + "weakly": 17541, + "cancel": 17542, + "polly": 17543, + "yielded": 17544, + "##emann": 17545, + "curly": 17546, + "prediction": 17547, + "battered": 17548, + "70s": 17549, + "vhs": 17550, + "jacqueline": 17551, + "render": 17552, + "sails": 17553, + "barked": 17554, + "detailing": 17555, + "grayson": 17556, + "riga": 17557, + "sloane": 17558, + "raging": 17559, + "##yah": 17560, + "herbs": 17561, + "bravo": 17562, + "##athlon": 17563, + "alloy": 17564, + "giggle": 17565, + "imminent": 17566, + "suffers": 17567, + "assumptions": 17568, + "waltz": 17569, + "##itate": 17570, + "accomplishments": 17571, + "##ited": 17572, + "bathing": 17573, + "remixed": 17574, + "deception": 17575, + "prefix": 17576, + "##emia": 17577, + "deepest": 17578, + "##tier": 17579, + "##eis": 17580, + "balkan": 17581, + "frogs": 17582, + "##rong": 17583, + "slab": 17584, + "##pate": 17585, + "philosophers": 17586, + "peterborough": 17587, + "grains": 17588, + "imports": 17589, + "dickinson": 17590, + "rwanda": 17591, + "##atics": 17592, + "1774": 17593, + "dirk": 17594, + "lan": 17595, + "tablets": 17596, + "##rove": 17597, + "clone": 17598, + "##rice": 17599, + "caretaker": 17600, + "hostilities": 17601, + "mclean": 17602, + "##gre": 17603, + "regimental": 17604, + "treasures": 17605, + "norms": 17606, + "impose": 17607, + "tsar": 17608, + "tango": 17609, + "diplomacy": 17610, + "variously": 17611, + "complain": 17612, + "192": 17613, + "recognise": 17614, + "arrests": 17615, + "1779": 17616, + "celestial": 17617, + "pulitzer": 17618, + "##dus": 17619, + "bing": 17620, + "libretto": 17621, + "##moor": 17622, + "adele": 17623, + "splash": 17624, + "##rite": 17625, + "expectation": 17626, + "lds": 17627, + "confronts": 17628, + "##izer": 17629, + "spontaneous": 17630, + "harmful": 17631, + "wedge": 17632, + "entrepreneurs": 17633, + "buyer": 17634, + "##ope": 17635, + "bilingual": 17636, + "translate": 17637, + "rugged": 17638, + "conner": 17639, + "circulated": 17640, + "uae": 17641, + "eaton": 17642, + "##gra": 17643, + "##zzle": 17644, + "lingered": 17645, + "lockheed": 17646, + "vishnu": 17647, + "reelection": 17648, + "alonso": 17649, + "##oom": 17650, + "joints": 17651, + "yankee": 17652, + "headline": 17653, + "cooperate": 17654, + "heinz": 17655, + "laureate": 17656, + "invading": 17657, + "##sford": 17658, + "echoes": 17659, + "scandinavian": 17660, + "##dham": 17661, + "hugging": 17662, + "vitamin": 17663, + "salute": 17664, + "micah": 17665, + "hind": 17666, + "trader": 17667, + "##sper": 17668, + "radioactive": 17669, + "##ndra": 17670, + "militants": 17671, + "poisoned": 17672, + "ratified": 17673, + "remark": 17674, + "campeonato": 17675, + "deprived": 17676, + "wander": 17677, + "prop": 17678, + "##dong": 17679, + "outlook": 17680, + "##tani": 17681, + "##rix": 17682, + "##eye": 17683, + "chiang": 17684, + "darcy": 17685, + "##oping": 17686, + "mandolin": 17687, + "spice": 17688, + "statesman": 17689, + "babylon": 17690, + "182": 17691, + "walled": 17692, + "forgetting": 17693, + "afro": 17694, + "##cap": 17695, + "158": 17696, + "giorgio": 17697, + "buffer": 17698, + "##polis": 17699, + "planetary": 17700, + "##gis": 17701, + "overlap": 17702, + "terminals": 17703, + "kinda": 17704, + "centenary": 17705, + "##bir": 17706, + "arising": 17707, + "manipulate": 17708, + "elm": 17709, + "ke": 17710, + "1770": 17711, + "ak": 17712, + "##tad": 17713, + "chrysler": 17714, + "mapped": 17715, + "moose": 17716, + "pomeranian": 17717, + "quad": 17718, + "macarthur": 17719, + "assemblies": 17720, + "shoreline": 17721, + "recalls": 17722, + "stratford": 17723, + "##rted": 17724, + "noticeable": 17725, + "##evic": 17726, + "imp": 17727, + "##rita": 17728, + "##sque": 17729, + "accustomed": 17730, + "supplying": 17731, + "tents": 17732, + "disgusted": 17733, + "vogue": 17734, + "sipped": 17735, + "filters": 17736, + "khz": 17737, + "reno": 17738, + "selecting": 17739, + "luftwaffe": 17740, + "mcmahon": 17741, + "tyne": 17742, + "masterpiece": 17743, + "carriages": 17744, + "collided": 17745, + "dunes": 17746, + "exercised": 17747, + "flare": 17748, + "remembers": 17749, + "muzzle": 17750, + "##mobile": 17751, + "heck": 17752, + "##rson": 17753, + "burgess": 17754, + "lunged": 17755, + "middleton": 17756, + "boycott": 17757, + "bilateral": 17758, + "##sity": 17759, + "hazardous": 17760, + "lumpur": 17761, + "multiplayer": 17762, + "spotlight": 17763, + "jackets": 17764, + "goldman": 17765, + "liege": 17766, + "porcelain": 17767, + "rag": 17768, + "waterford": 17769, + "benz": 17770, + "attracts": 17771, + "hopeful": 17772, + "battling": 17773, + "ottomans": 17774, + "kensington": 17775, + "baked": 17776, + "hymns": 17777, + "cheyenne": 17778, + "lattice": 17779, + "levine": 17780, + "borrow": 17781, + "polymer": 17782, + "clashes": 17783, + "michaels": 17784, + "monitored": 17785, + "commitments": 17786, + "denounced": 17787, + "##25": 17788, + "##von": 17789, + "cavity": 17790, + "##oney": 17791, + "hobby": 17792, + "akin": 17793, + "##holders": 17794, + "futures": 17795, + "intricate": 17796, + "cornish": 17797, + "patty": 17798, + "##oned": 17799, + "illegally": 17800, + "dolphin": 17801, + "##lag": 17802, + "barlow": 17803, + "yellowish": 17804, + "maddie": 17805, + "apologized": 17806, + "luton": 17807, + "plagued": 17808, + "##puram": 17809, + "nana": 17810, + "##rds": 17811, + "sway": 17812, + "fanny": 17813, + "łodz": 17814, + "##rino": 17815, + "psi": 17816, + "suspicions": 17817, + "hanged": 17818, + "##eding": 17819, + "initiate": 17820, + "charlton": 17821, + "##por": 17822, + "nak": 17823, + "competent": 17824, + "235": 17825, + "analytical": 17826, + "annex": 17827, + "wardrobe": 17828, + "reservations": 17829, + "##rma": 17830, + "sect": 17831, + "162": 17832, + "fairfax": 17833, + "hedge": 17834, + "piled": 17835, + "buckingham": 17836, + "uneven": 17837, + "bauer": 17838, + "simplicity": 17839, + "snyder": 17840, + "interpret": 17841, + "accountability": 17842, + "donors": 17843, + "moderately": 17844, + "byrd": 17845, + "continents": 17846, + "##cite": 17847, + "##max": 17848, + "disciple": 17849, + "hr": 17850, + "jamaican": 17851, + "ping": 17852, + "nominees": 17853, + "##uss": 17854, + "mongolian": 17855, + "diver": 17856, + "attackers": 17857, + "eagerly": 17858, + "ideological": 17859, + "pillows": 17860, + "miracles": 17861, + "apartheid": 17862, + "revolver": 17863, + "sulfur": 17864, + "clinics": 17865, + "moran": 17866, + "163": 17867, + "##enko": 17868, + "ile": 17869, + "katy": 17870, + "rhetoric": 17871, + "##icated": 17872, + "chronology": 17873, + "recycling": 17874, + "##hrer": 17875, + "elongated": 17876, + "mughal": 17877, + "pascal": 17878, + "profiles": 17879, + "vibration": 17880, + "databases": 17881, + "domination": 17882, + "##fare": 17883, + "##rant": 17884, + "matthias": 17885, + "digest": 17886, + "rehearsal": 17887, + "polling": 17888, + "weiss": 17889, + "initiation": 17890, + "reeves": 17891, + "clinging": 17892, + "flourished": 17893, + "impress": 17894, + "ngo": 17895, + "##hoff": 17896, + "##ume": 17897, + "buckley": 17898, + "symposium": 17899, + "rhythms": 17900, + "weed": 17901, + "emphasize": 17902, + "transforming": 17903, + "##taking": 17904, + "##gence": 17905, + "##yman": 17906, + "accountant": 17907, + "analyze": 17908, + "flicker": 17909, + "foil": 17910, + "priesthood": 17911, + "voluntarily": 17912, + "decreases": 17913, + "##80": 17914, + "##hya": 17915, + "slater": 17916, + "sv": 17917, + "charting": 17918, + "mcgill": 17919, + "##lde": 17920, + "moreno": 17921, + "##iu": 17922, + "besieged": 17923, + "zur": 17924, + "robes": 17925, + "##phic": 17926, + "admitting": 17927, + "api": 17928, + "deported": 17929, + "turmoil": 17930, + "peyton": 17931, + "earthquakes": 17932, + "##ares": 17933, + "nationalists": 17934, + "beau": 17935, + "clair": 17936, + "brethren": 17937, + "interrupt": 17938, + "welch": 17939, + "curated": 17940, + "galerie": 17941, + "requesting": 17942, + "164": 17943, + "##ested": 17944, + "impending": 17945, + "steward": 17946, + "viper": 17947, + "##vina": 17948, + "complaining": 17949, + "beautifully": 17950, + "brandy": 17951, + "foam": 17952, + "nl": 17953, + "1660": 17954, + "##cake": 17955, + "alessandro": 17956, + "punches": 17957, + "laced": 17958, + "explanations": 17959, + "##lim": 17960, + "attribute": 17961, + "clit": 17962, + "reggie": 17963, + "discomfort": 17964, + "##cards": 17965, + "smoothed": 17966, + "whales": 17967, + "##cene": 17968, + "adler": 17969, + "countered": 17970, + "duffy": 17971, + "disciplinary": 17972, + "widening": 17973, + "recipe": 17974, + "reliance": 17975, + "conducts": 17976, + "goats": 17977, + "gradient": 17978, + "preaching": 17979, + "##shaw": 17980, + "matilda": 17981, + "quasi": 17982, + "striped": 17983, + "meridian": 17984, + "cannabis": 17985, + "cordoba": 17986, + "certificates": 17987, + "##agh": 17988, + "##tering": 17989, + "graffiti": 17990, + "hangs": 17991, + "pilgrims": 17992, + "repeats": 17993, + "##ych": 17994, + "revive": 17995, + "urine": 17996, + "etat": 17997, + "##hawk": 17998, + "fueled": 17999, + "belts": 18000, + "fuzzy": 18001, + "susceptible": 18002, + "##hang": 18003, + "mauritius": 18004, + "salle": 18005, + "sincere": 18006, + "beers": 18007, + "hooks": 18008, + "##cki": 18009, + "arbitration": 18010, + "entrusted": 18011, + "advise": 18012, + "sniffed": 18013, + "seminar": 18014, + "junk": 18015, + "donnell": 18016, + "processors": 18017, + "principality": 18018, + "strapped": 18019, + "celia": 18020, + "mendoza": 18021, + "everton": 18022, + "fortunes": 18023, + "prejudice": 18024, + "starving": 18025, + "reassigned": 18026, + "steamer": 18027, + "##lund": 18028, + "tuck": 18029, + "evenly": 18030, + "foreman": 18031, + "##ffen": 18032, + "dans": 18033, + "375": 18034, + "envisioned": 18035, + "slit": 18036, + "##xy": 18037, + "baseman": 18038, + "liberia": 18039, + "rosemary": 18040, + "##weed": 18041, + "electrified": 18042, + "periodically": 18043, + "potassium": 18044, + "stride": 18045, + "contexts": 18046, + "sperm": 18047, + "slade": 18048, + "mariners": 18049, + "influx": 18050, + "bianca": 18051, + "subcommittee": 18052, + "##rane": 18053, + "spilling": 18054, + "icao": 18055, + "estuary": 18056, + "##nock": 18057, + "delivers": 18058, + "iphone": 18059, + "##ulata": 18060, + "isa": 18061, + "mira": 18062, + "bohemian": 18063, + "dessert": 18064, + "##sbury": 18065, + "welcoming": 18066, + "proudly": 18067, + "slowing": 18068, + "##chs": 18069, + "musee": 18070, + "ascension": 18071, + "russ": 18072, + "##vian": 18073, + "waits": 18074, + "##psy": 18075, + "africans": 18076, + "exploit": 18077, + "##morphic": 18078, + "gov": 18079, + "eccentric": 18080, + "crab": 18081, + "peck": 18082, + "##ull": 18083, + "entrances": 18084, + "formidable": 18085, + "marketplace": 18086, + "groom": 18087, + "bolted": 18088, + "metabolism": 18089, + "patton": 18090, + "robbins": 18091, + "courier": 18092, + "payload": 18093, + "endure": 18094, + "##ifier": 18095, + "andes": 18096, + "refrigerator": 18097, + "##pr": 18098, + "ornate": 18099, + "##uca": 18100, + "ruthless": 18101, + "illegitimate": 18102, + "masonry": 18103, + "strasbourg": 18104, + "bikes": 18105, + "adobe": 18106, + "##³": 18107, + "apples": 18108, + "quintet": 18109, + "willingly": 18110, + "niche": 18111, + "bakery": 18112, + "corpses": 18113, + "energetic": 18114, + "##cliffe": 18115, + "##sser": 18116, + "##ards": 18117, + "177": 18118, + "centimeters": 18119, + "centro": 18120, + "fuscous": 18121, + "cretaceous": 18122, + "rancho": 18123, + "##yde": 18124, + "andrei": 18125, + "telecom": 18126, + "tottenham": 18127, + "oasis": 18128, + "ordination": 18129, + "vulnerability": 18130, + "presiding": 18131, + "corey": 18132, + "cp": 18133, + "penguins": 18134, + "sims": 18135, + "##pis": 18136, + "malawi": 18137, + "piss": 18138, + "##48": 18139, + "correction": 18140, + "##cked": 18141, + "##ffle": 18142, + "##ryn": 18143, + "countdown": 18144, + "detectives": 18145, + "psychiatrist": 18146, + "psychedelic": 18147, + "dinosaurs": 18148, + "blouse": 18149, + "##get": 18150, + "choi": 18151, + "vowed": 18152, + "##oz": 18153, + "randomly": 18154, + "##pol": 18155, + "49ers": 18156, + "scrub": 18157, + "blanche": 18158, + "bruins": 18159, + "dusseldorf": 18160, + "##using": 18161, + "unwanted": 18162, + "##ums": 18163, + "212": 18164, + "dominique": 18165, + "elevations": 18166, + "headlights": 18167, + "om": 18168, + "laguna": 18169, + "##oga": 18170, + "1750": 18171, + "famously": 18172, + "ignorance": 18173, + "shrewsbury": 18174, + "##aine": 18175, + "ajax": 18176, + "breuning": 18177, + "che": 18178, + "confederacy": 18179, + "greco": 18180, + "overhaul": 18181, + "##screen": 18182, + "paz": 18183, + "skirts": 18184, + "disagreement": 18185, + "cruelty": 18186, + "jagged": 18187, + "phoebe": 18188, + "shifter": 18189, + "hovered": 18190, + "viruses": 18191, + "##wes": 18192, + "mandy": 18193, + "##lined": 18194, + "##gc": 18195, + "landlord": 18196, + "squirrel": 18197, + "dashed": 18198, + "##ι": 18199, + "ornamental": 18200, + "gag": 18201, + "wally": 18202, + "grange": 18203, + "literal": 18204, + "spurs": 18205, + "undisclosed": 18206, + "proceeding": 18207, + "yin": 18208, + "##text": 18209, + "billie": 18210, + "orphan": 18211, + "spanned": 18212, + "humidity": 18213, + "indy": 18214, + "weighted": 18215, + "presentations": 18216, + "explosions": 18217, + "lucian": 18218, + "##tary": 18219, + "vaughn": 18220, + "hindus": 18221, + "##anga": 18222, + "##hell": 18223, + "psycho": 18224, + "171": 18225, + "daytona": 18226, + "protects": 18227, + "efficiently": 18228, + "rematch": 18229, + "sly": 18230, + "tandem": 18231, + "##oya": 18232, + "rebranded": 18233, + "impaired": 18234, + "hee": 18235, + "metropolis": 18236, + "peach": 18237, + "godfrey": 18238, + "diaspora": 18239, + "ethnicity": 18240, + "prosperous": 18241, + "gleaming": 18242, + "dar": 18243, + "grossing": 18244, + "playback": 18245, + "##rden": 18246, + "stripe": 18247, + "pistols": 18248, + "##tain": 18249, + "births": 18250, + "labelled": 18251, + "##cating": 18252, + "172": 18253, + "rudy": 18254, + "alba": 18255, + "##onne": 18256, + "aquarium": 18257, + "hostility": 18258, + "##gb": 18259, + "##tase": 18260, + "shudder": 18261, + "sumatra": 18262, + "hardest": 18263, + "lakers": 18264, + "consonant": 18265, + "creeping": 18266, + "demos": 18267, + "homicide": 18268, + "capsule": 18269, + "zeke": 18270, + "liberties": 18271, + "expulsion": 18272, + "pueblo": 18273, + "##comb": 18274, + "trait": 18275, + "transporting": 18276, + "##ddin": 18277, + "##neck": 18278, + "##yna": 18279, + "depart": 18280, + "gregg": 18281, + "mold": 18282, + "ledge": 18283, + "hangar": 18284, + "oldham": 18285, + "playboy": 18286, + "termination": 18287, + "analysts": 18288, + "gmbh": 18289, + "romero": 18290, + "##itic": 18291, + "insist": 18292, + "cradle": 18293, + "filthy": 18294, + "brightness": 18295, + "slash": 18296, + "shootout": 18297, + "deposed": 18298, + "bordering": 18299, + "##truct": 18300, + "isis": 18301, + "microwave": 18302, + "tumbled": 18303, + "sheltered": 18304, + "cathy": 18305, + "werewolves": 18306, + "messy": 18307, + "andersen": 18308, + "convex": 18309, + "clapped": 18310, + "clinched": 18311, + "satire": 18312, + "wasting": 18313, + "edo": 18314, + "vc": 18315, + "rufus": 18316, + "##jak": 18317, + "mont": 18318, + "##etti": 18319, + "poznan": 18320, + "##keeping": 18321, + "restructuring": 18322, + "transverse": 18323, + "##rland": 18324, + "azerbaijani": 18325, + "slovene": 18326, + "gestures": 18327, + "roommate": 18328, + "choking": 18329, + "shear": 18330, + "##quist": 18331, + "vanguard": 18332, + "oblivious": 18333, + "##hiro": 18334, + "disagreed": 18335, + "baptism": 18336, + "##lich": 18337, + "coliseum": 18338, + "##aceae": 18339, + "salvage": 18340, + "societe": 18341, + "cory": 18342, + "locke": 18343, + "relocation": 18344, + "relying": 18345, + "versailles": 18346, + "ahl": 18347, + "swelling": 18348, + "##elo": 18349, + "cheerful": 18350, + "##word": 18351, + "##edes": 18352, + "gin": 18353, + "sarajevo": 18354, + "obstacle": 18355, + "diverted": 18356, + "##nac": 18357, + "messed": 18358, + "thoroughbred": 18359, + "fluttered": 18360, + "utrecht": 18361, + "chewed": 18362, + "acquaintance": 18363, + "assassins": 18364, + "dispatch": 18365, + "mirza": 18366, + "##wart": 18367, + "nike": 18368, + "salzburg": 18369, + "swell": 18370, + "yen": 18371, + "##gee": 18372, + "idle": 18373, + "ligue": 18374, + "samson": 18375, + "##nds": 18376, + "##igh": 18377, + "playful": 18378, + "spawned": 18379, + "##cise": 18380, + "tease": 18381, + "##case": 18382, + "burgundy": 18383, + "##bot": 18384, + "stirring": 18385, + "skeptical": 18386, + "interceptions": 18387, + "marathi": 18388, + "##dies": 18389, + "bedrooms": 18390, + "aroused": 18391, + "pinch": 18392, + "##lik": 18393, + "preferences": 18394, + "tattoos": 18395, + "buster": 18396, + "digitally": 18397, + "projecting": 18398, + "rust": 18399, + "##ital": 18400, + "kitten": 18401, + "priorities": 18402, + "addison": 18403, + "pseudo": 18404, + "##guard": 18405, + "dusk": 18406, + "icons": 18407, + "sermon": 18408, + "##psis": 18409, + "##iba": 18410, + "bt": 18411, + "##lift": 18412, + "##xt": 18413, + "ju": 18414, + "truce": 18415, + "rink": 18416, + "##dah": 18417, + "##wy": 18418, + "defects": 18419, + "psychiatry": 18420, + "offences": 18421, + "calculate": 18422, + "glucose": 18423, + "##iful": 18424, + "##rized": 18425, + "##unda": 18426, + "francaise": 18427, + "##hari": 18428, + "richest": 18429, + "warwickshire": 18430, + "carly": 18431, + "1763": 18432, + "purity": 18433, + "redemption": 18434, + "lending": 18435, + "##cious": 18436, + "muse": 18437, + "bruises": 18438, + "cerebral": 18439, + "aero": 18440, + "carving": 18441, + "##name": 18442, + "preface": 18443, + "terminology": 18444, + "invade": 18445, + "monty": 18446, + "##int": 18447, + "anarchist": 18448, + "blurred": 18449, + "##iled": 18450, + "rossi": 18451, + "treats": 18452, + "guts": 18453, + "shu": 18454, + "foothills": 18455, + "ballads": 18456, + "undertaking": 18457, + "premise": 18458, + "cecilia": 18459, + "affiliates": 18460, + "blasted": 18461, + "conditional": 18462, + "wilder": 18463, + "minors": 18464, + "drone": 18465, + "rudolph": 18466, + "buffy": 18467, + "swallowing": 18468, + "horton": 18469, + "attested": 18470, + "##hop": 18471, + "rutherford": 18472, + "howell": 18473, + "primetime": 18474, + "livery": 18475, + "penal": 18476, + "##bis": 18477, + "minimize": 18478, + "hydro": 18479, + "wrecked": 18480, + "wrought": 18481, + "palazzo": 18482, + "##gling": 18483, + "cans": 18484, + "vernacular": 18485, + "friedman": 18486, + "nobleman": 18487, + "shale": 18488, + "walnut": 18489, + "danielle": 18490, + "##ection": 18491, + "##tley": 18492, + "sears": 18493, + "##kumar": 18494, + "chords": 18495, + "lend": 18496, + "flipping": 18497, + "streamed": 18498, + "por": 18499, + "dracula": 18500, + "gallons": 18501, + "sacrifices": 18502, + "gamble": 18503, + "orphanage": 18504, + "##iman": 18505, + "mckenzie": 18506, + "##gible": 18507, + "boxers": 18508, + "daly": 18509, + "##balls": 18510, + "##ان": 18511, + "208": 18512, + "##ific": 18513, + "##rative": 18514, + "##iq": 18515, + "exploited": 18516, + "slated": 18517, + "##uity": 18518, + "circling": 18519, + "hillary": 18520, + "pinched": 18521, + "goldberg": 18522, + "provost": 18523, + "campaigning": 18524, + "lim": 18525, + "piles": 18526, + "ironically": 18527, + "jong": 18528, + "mohan": 18529, + "successors": 18530, + "usaf": 18531, + "##tem": 18532, + "##ught": 18533, + "autobiographical": 18534, + "haute": 18535, + "preserves": 18536, + "##ending": 18537, + "acquitted": 18538, + "comparisons": 18539, + "203": 18540, + "hydroelectric": 18541, + "gangs": 18542, + "cypriot": 18543, + "torpedoes": 18544, + "rushes": 18545, + "chrome": 18546, + "derive": 18547, + "bumps": 18548, + "instability": 18549, + "fiat": 18550, + "pets": 18551, + "##mbe": 18552, + "silas": 18553, + "dye": 18554, + "reckless": 18555, + "settler": 18556, + "##itation": 18557, + "info": 18558, + "heats": 18559, + "##writing": 18560, + "176": 18561, + "canonical": 18562, + "maltese": 18563, + "fins": 18564, + "mushroom": 18565, + "stacy": 18566, + "aspen": 18567, + "avid": 18568, + "##kur": 18569, + "##loading": 18570, + "vickers": 18571, + "gaston": 18572, + "hillside": 18573, + "statutes": 18574, + "wilde": 18575, + "gail": 18576, + "kung": 18577, + "sabine": 18578, + "comfortably": 18579, + "motorcycles": 18580, + "##rgo": 18581, + "169": 18582, + "pneumonia": 18583, + "fetch": 18584, + "##sonic": 18585, + "axel": 18586, + "faintly": 18587, + "parallels": 18588, + "##oop": 18589, + "mclaren": 18590, + "spouse": 18591, + "compton": 18592, + "interdisciplinary": 18593, + "miner": 18594, + "##eni": 18595, + "181": 18596, + "clamped": 18597, + "##chal": 18598, + "##llah": 18599, + "separates": 18600, + "versa": 18601, + "##mler": 18602, + "scarborough": 18603, + "labrador": 18604, + "##lity": 18605, + "##osing": 18606, + "rutgers": 18607, + "hurdles": 18608, + "como": 18609, + "166": 18610, + "burt": 18611, + "divers": 18612, + "##100": 18613, + "wichita": 18614, + "cade": 18615, + "coincided": 18616, + "##erson": 18617, + "bruised": 18618, + "mla": 18619, + "##pper": 18620, + "vineyard": 18621, + "##ili": 18622, + "##brush": 18623, + "notch": 18624, + "mentioning": 18625, + "jase": 18626, + "hearted": 18627, + "kits": 18628, + "doe": 18629, + "##acle": 18630, + "pomerania": 18631, + "##ady": 18632, + "ronan": 18633, + "seizure": 18634, + "pavel": 18635, + "problematic": 18636, + "##zaki": 18637, + "domenico": 18638, + "##ulin": 18639, + "catering": 18640, + "penelope": 18641, + "dependence": 18642, + "parental": 18643, + "emilio": 18644, + "ministerial": 18645, + "atkinson": 18646, + "##bolic": 18647, + "clarkson": 18648, + "chargers": 18649, + "colby": 18650, + "grill": 18651, + "peeked": 18652, + "arises": 18653, + "summon": 18654, + "##aged": 18655, + "fools": 18656, + "##grapher": 18657, + "faculties": 18658, + "qaeda": 18659, + "##vial": 18660, + "garner": 18661, + "refurbished": 18662, + "##hwa": 18663, + "geelong": 18664, + "disasters": 18665, + "nudged": 18666, + "bs": 18667, + "shareholder": 18668, + "lori": 18669, + "algae": 18670, + "reinstated": 18671, + "rot": 18672, + "##ades": 18673, + "##nous": 18674, + "invites": 18675, + "stainless": 18676, + "183": 18677, + "inclusive": 18678, + "##itude": 18679, + "diocesan": 18680, + "til": 18681, + "##icz": 18682, + "denomination": 18683, + "##xa": 18684, + "benton": 18685, + "floral": 18686, + "registers": 18687, + "##ider": 18688, + "##erman": 18689, + "##kell": 18690, + "absurd": 18691, + "brunei": 18692, + "guangzhou": 18693, + "hitter": 18694, + "retaliation": 18695, + "##uled": 18696, + "##eve": 18697, + "blanc": 18698, + "nh": 18699, + "consistency": 18700, + "contamination": 18701, + "##eres": 18702, + "##rner": 18703, + "dire": 18704, + "palermo": 18705, + "broadcasters": 18706, + "diaries": 18707, + "inspire": 18708, + "vols": 18709, + "brewer": 18710, + "tightening": 18711, + "ky": 18712, + "mixtape": 18713, + "hormone": 18714, + "##tok": 18715, + "stokes": 18716, + "##color": 18717, + "##dly": 18718, + "##ssi": 18719, + "pg": 18720, + "##ometer": 18721, + "##lington": 18722, + "sanitation": 18723, + "##tility": 18724, + "intercontinental": 18725, + "apps": 18726, + "##adt": 18727, + "¹⁄₂": 18728, + "cylinders": 18729, + "economies": 18730, + "favourable": 18731, + "unison": 18732, + "croix": 18733, + "gertrude": 18734, + "odyssey": 18735, + "vanity": 18736, + "dangling": 18737, + "##logists": 18738, + "upgrades": 18739, + "dice": 18740, + "middleweight": 18741, + "practitioner": 18742, + "##ight": 18743, + "206": 18744, + "henrik": 18745, + "parlor": 18746, + "orion": 18747, + "angered": 18748, + "lac": 18749, + "python": 18750, + "blurted": 18751, + "##rri": 18752, + "sensual": 18753, + "intends": 18754, + "swings": 18755, + "angled": 18756, + "##phs": 18757, + "husky": 18758, + "attain": 18759, + "peerage": 18760, + "precinct": 18761, + "textiles": 18762, + "cheltenham": 18763, + "shuffled": 18764, + "dai": 18765, + "confess": 18766, + "tasting": 18767, + "bhutan": 18768, + "##riation": 18769, + "tyrone": 18770, + "segregation": 18771, + "abrupt": 18772, + "ruiz": 18773, + "##rish": 18774, + "smirked": 18775, + "blackwell": 18776, + "confidential": 18777, + "browning": 18778, + "amounted": 18779, + "##put": 18780, + "vase": 18781, + "scarce": 18782, + "fabulous": 18783, + "raided": 18784, + "staple": 18785, + "guyana": 18786, + "unemployed": 18787, + "glider": 18788, + "shay": 18789, + "##tow": 18790, + "carmine": 18791, + "troll": 18792, + "intervene": 18793, + "squash": 18794, + "superstar": 18795, + "##uce": 18796, + "cylindrical": 18797, + "len": 18798, + "roadway": 18799, + "researched": 18800, + "handy": 18801, + "##rium": 18802, + "##jana": 18803, + "meta": 18804, + "lao": 18805, + "declares": 18806, + "##rring": 18807, + "##tadt": 18808, + "##elin": 18809, + "##kova": 18810, + "willem": 18811, + "shrubs": 18812, + "napoleonic": 18813, + "realms": 18814, + "skater": 18815, + "qi": 18816, + "volkswagen": 18817, + "##ł": 18818, + "tad": 18819, + "hara": 18820, + "archaeologist": 18821, + "awkwardly": 18822, + "eerie": 18823, + "##kind": 18824, + "wiley": 18825, + "##heimer": 18826, + "##24": 18827, + "titus": 18828, + "organizers": 18829, + "cfl": 18830, + "crusaders": 18831, + "lama": 18832, + "usb": 18833, + "vent": 18834, + "enraged": 18835, + "thankful": 18836, + "occupants": 18837, + "maximilian": 18838, + "##gaard": 18839, + "possessing": 18840, + "textbooks": 18841, + "##oran": 18842, + "collaborator": 18843, + "quaker": 18844, + "##ulo": 18845, + "avalanche": 18846, + "mono": 18847, + "silky": 18848, + "straits": 18849, + "isaiah": 18850, + "mustang": 18851, + "surged": 18852, + "resolutions": 18853, + "potomac": 18854, + "descend": 18855, + "cl": 18856, + "kilograms": 18857, + "plato": 18858, + "strains": 18859, + "saturdays": 18860, + "##olin": 18861, + "bernstein": 18862, + "##ype": 18863, + "holstein": 18864, + "ponytail": 18865, + "##watch": 18866, + "belize": 18867, + "conversely": 18868, + "heroine": 18869, + "perpetual": 18870, + "##ylus": 18871, + "charcoal": 18872, + "piedmont": 18873, + "glee": 18874, + "negotiating": 18875, + "backdrop": 18876, + "prologue": 18877, + "##jah": 18878, + "##mmy": 18879, + "pasadena": 18880, + "climbs": 18881, + "ramos": 18882, + "sunni": 18883, + "##holm": 18884, + "##tner": 18885, + "##tri": 18886, + "anand": 18887, + "deficiency": 18888, + "hertfordshire": 18889, + "stout": 18890, + "##avi": 18891, + "aperture": 18892, + "orioles": 18893, + "##irs": 18894, + "doncaster": 18895, + "intrigued": 18896, + "bombed": 18897, + "coating": 18898, + "otis": 18899, + "##mat": 18900, + "cocktail": 18901, + "##jit": 18902, + "##eto": 18903, + "amir": 18904, + "arousal": 18905, + "sar": 18906, + "##proof": 18907, + "##act": 18908, + "##ories": 18909, + "dixie": 18910, + "pots": 18911, + "##bow": 18912, + "whereabouts": 18913, + "159": 18914, + "##fted": 18915, + "drains": 18916, + "bullying": 18917, + "cottages": 18918, + "scripture": 18919, + "coherent": 18920, + "fore": 18921, + "poe": 18922, + "appetite": 18923, + "##uration": 18924, + "sampled": 18925, + "##ators": 18926, + "##dp": 18927, + "derrick": 18928, + "rotor": 18929, + "jays": 18930, + "peacock": 18931, + "installment": 18932, + "##rro": 18933, + "advisors": 18934, + "##coming": 18935, + "rodeo": 18936, + "scotch": 18937, + "##mot": 18938, + "##db": 18939, + "##fen": 18940, + "##vant": 18941, + "ensued": 18942, + "rodrigo": 18943, + "dictatorship": 18944, + "martyrs": 18945, + "twenties": 18946, + "##н": 18947, + "towed": 18948, + "incidence": 18949, + "marta": 18950, + "rainforest": 18951, + "sai": 18952, + "scaled": 18953, + "##cles": 18954, + "oceanic": 18955, + "qualifiers": 18956, + "symphonic": 18957, + "mcbride": 18958, + "dislike": 18959, + "generalized": 18960, + "aubrey": 18961, + "colonization": 18962, + "##iation": 18963, + "##lion": 18964, + "##ssing": 18965, + "disliked": 18966, + "lublin": 18967, + "salesman": 18968, + "##ulates": 18969, + "spherical": 18970, + "whatsoever": 18971, + "sweating": 18972, + "avalon": 18973, + "contention": 18974, + "punt": 18975, + "severity": 18976, + "alderman": 18977, + "atari": 18978, + "##dina": 18979, + "##grant": 18980, + "##rop": 18981, + "scarf": 18982, + "seville": 18983, + "vertices": 18984, + "annexation": 18985, + "fairfield": 18986, + "fascination": 18987, + "inspiring": 18988, + "launches": 18989, + "palatinate": 18990, + "regretted": 18991, + "##rca": 18992, + "feral": 18993, + "##iom": 18994, + "elk": 18995, + "nap": 18996, + "olsen": 18997, + "reddy": 18998, + "yong": 18999, + "##leader": 19000, + "##iae": 19001, + "garment": 19002, + "transports": 19003, + "feng": 19004, + "gracie": 19005, + "outrage": 19006, + "viceroy": 19007, + "insides": 19008, + "##esis": 19009, + "breakup": 19010, + "grady": 19011, + "organizer": 19012, + "softer": 19013, + "grimaced": 19014, + "222": 19015, + "murals": 19016, + "galicia": 19017, + "arranging": 19018, + "vectors": 19019, + "##rsten": 19020, + "bas": 19021, + "##sb": 19022, + "##cens": 19023, + "sloan": 19024, + "##eka": 19025, + "bitten": 19026, + "ara": 19027, + "fender": 19028, + "nausea": 19029, + "bumped": 19030, + "kris": 19031, + "banquet": 19032, + "comrades": 19033, + "detector": 19034, + "persisted": 19035, + "##llan": 19036, + "adjustment": 19037, + "endowed": 19038, + "cinemas": 19039, + "##shot": 19040, + "sellers": 19041, + "##uman": 19042, + "peek": 19043, + "epa": 19044, + "kindly": 19045, + "neglect": 19046, + "simpsons": 19047, + "talon": 19048, + "mausoleum": 19049, + "runaway": 19050, + "hangul": 19051, + "lookout": 19052, + "##cic": 19053, + "rewards": 19054, + "coughed": 19055, + "acquainted": 19056, + "chloride": 19057, + "##ald": 19058, + "quicker": 19059, + "accordion": 19060, + "neolithic": 19061, + "##qa": 19062, + "artemis": 19063, + "coefficient": 19064, + "lenny": 19065, + "pandora": 19066, + "tx": 19067, + "##xed": 19068, + "ecstasy": 19069, + "litter": 19070, + "segunda": 19071, + "chairperson": 19072, + "gemma": 19073, + "hiss": 19074, + "rumor": 19075, + "vow": 19076, + "nasal": 19077, + "antioch": 19078, + "compensate": 19079, + "patiently": 19080, + "transformers": 19081, + "##eded": 19082, + "judo": 19083, + "morrow": 19084, + "penis": 19085, + "posthumous": 19086, + "philips": 19087, + "bandits": 19088, + "husbands": 19089, + "denote": 19090, + "flaming": 19091, + "##any": 19092, + "##phones": 19093, + "langley": 19094, + "yorker": 19095, + "1760": 19096, + "walters": 19097, + "##uo": 19098, + "##kle": 19099, + "gubernatorial": 19100, + "fatty": 19101, + "samsung": 19102, + "leroy": 19103, + "outlaw": 19104, + "##nine": 19105, + "unpublished": 19106, + "poole": 19107, + "jakob": 19108, + "##ᵢ": 19109, + "##ₙ": 19110, + "crete": 19111, + "distorted": 19112, + "superiority": 19113, + "##dhi": 19114, + "intercept": 19115, + "crust": 19116, + "mig": 19117, + "claus": 19118, + "crashes": 19119, + "positioning": 19120, + "188": 19121, + "stallion": 19122, + "301": 19123, + "frontal": 19124, + "armistice": 19125, + "##estinal": 19126, + "elton": 19127, + "aj": 19128, + "encompassing": 19129, + "camel": 19130, + "commemorated": 19131, + "malaria": 19132, + "woodward": 19133, + "calf": 19134, + "cigar": 19135, + "penetrate": 19136, + "##oso": 19137, + "willard": 19138, + "##rno": 19139, + "##uche": 19140, + "illustrate": 19141, + "amusing": 19142, + "convergence": 19143, + "noteworthy": 19144, + "##lma": 19145, + "##rva": 19146, + "journeys": 19147, + "realise": 19148, + "manfred": 19149, + "##sable": 19150, + "410": 19151, + "##vocation": 19152, + "hearings": 19153, + "fiance": 19154, + "##posed": 19155, + "educators": 19156, + "provoked": 19157, + "adjusting": 19158, + "##cturing": 19159, + "modular": 19160, + "stockton": 19161, + "paterson": 19162, + "vlad": 19163, + "rejects": 19164, + "electors": 19165, + "selena": 19166, + "maureen": 19167, + "##tres": 19168, + "uber": 19169, + "##rce": 19170, + "swirled": 19171, + "##num": 19172, + "proportions": 19173, + "nanny": 19174, + "pawn": 19175, + "naturalist": 19176, + "parma": 19177, + "apostles": 19178, + "awoke": 19179, + "ethel": 19180, + "wen": 19181, + "##bey": 19182, + "monsoon": 19183, + "overview": 19184, + "##inating": 19185, + "mccain": 19186, + "rendition": 19187, + "risky": 19188, + "adorned": 19189, + "##ih": 19190, + "equestrian": 19191, + "germain": 19192, + "nj": 19193, + "conspicuous": 19194, + "confirming": 19195, + "##yoshi": 19196, + "shivering": 19197, + "##imeter": 19198, + "milestone": 19199, + "rumours": 19200, + "flinched": 19201, + "bounds": 19202, + "smacked": 19203, + "token": 19204, + "##bei": 19205, + "lectured": 19206, + "automobiles": 19207, + "##shore": 19208, + "impacted": 19209, + "##iable": 19210, + "nouns": 19211, + "nero": 19212, + "##leaf": 19213, + "ismail": 19214, + "prostitute": 19215, + "trams": 19216, + "##lace": 19217, + "bridget": 19218, + "sud": 19219, + "stimulus": 19220, + "impressions": 19221, + "reins": 19222, + "revolves": 19223, + "##oud": 19224, + "##gned": 19225, + "giro": 19226, + "honeymoon": 19227, + "##swell": 19228, + "criterion": 19229, + "##sms": 19230, + "##uil": 19231, + "libyan": 19232, + "prefers": 19233, + "##osition": 19234, + "211": 19235, + "preview": 19236, + "sucks": 19237, + "accusation": 19238, + "bursts": 19239, + "metaphor": 19240, + "diffusion": 19241, + "tolerate": 19242, + "faye": 19243, + "betting": 19244, + "cinematographer": 19245, + "liturgical": 19246, + "specials": 19247, + "bitterly": 19248, + "humboldt": 19249, + "##ckle": 19250, + "flux": 19251, + "rattled": 19252, + "##itzer": 19253, + "archaeologists": 19254, + "odor": 19255, + "authorised": 19256, + "marshes": 19257, + "discretion": 19258, + "##ов": 19259, + "alarmed": 19260, + "archaic": 19261, + "inverse": 19262, + "##leton": 19263, + "explorers": 19264, + "##pine": 19265, + "drummond": 19266, + "tsunami": 19267, + "woodlands": 19268, + "##minate": 19269, + "##tland": 19270, + "booklet": 19271, + "insanity": 19272, + "owning": 19273, + "insert": 19274, + "crafted": 19275, + "calculus": 19276, + "##tore": 19277, + "receivers": 19278, + "##bt": 19279, + "stung": 19280, + "##eca": 19281, + "##nched": 19282, + "prevailing": 19283, + "travellers": 19284, + "eyeing": 19285, + "lila": 19286, + "graphs": 19287, + "##borne": 19288, + "178": 19289, + "julien": 19290, + "##won": 19291, + "morale": 19292, + "adaptive": 19293, + "therapist": 19294, + "erica": 19295, + "cw": 19296, + "libertarian": 19297, + "bowman": 19298, + "pitches": 19299, + "vita": 19300, + "##ional": 19301, + "crook": 19302, + "##ads": 19303, + "##entation": 19304, + "caledonia": 19305, + "mutiny": 19306, + "##sible": 19307, + "1840s": 19308, + "automation": 19309, + "##ß": 19310, + "flock": 19311, + "##pia": 19312, + "ironic": 19313, + "pathology": 19314, + "##imus": 19315, + "remarried": 19316, + "##22": 19317, + "joker": 19318, + "withstand": 19319, + "energies": 19320, + "##att": 19321, + "shropshire": 19322, + "hostages": 19323, + "madeleine": 19324, + "tentatively": 19325, + "conflicting": 19326, + "mateo": 19327, + "recipes": 19328, + "euros": 19329, + "ol": 19330, + "mercenaries": 19331, + "nico": 19332, + "##ndon": 19333, + "albuquerque": 19334, + "augmented": 19335, + "mythical": 19336, + "bel": 19337, + "freud": 19338, + "##child": 19339, + "cough": 19340, + "##lica": 19341, + "365": 19342, + "freddy": 19343, + "lillian": 19344, + "genetically": 19345, + "nuremberg": 19346, + "calder": 19347, + "209": 19348, + "bonn": 19349, + "outdoors": 19350, + "paste": 19351, + "suns": 19352, + "urgency": 19353, + "vin": 19354, + "restraint": 19355, + "tyson": 19356, + "##cera": 19357, + "##selle": 19358, + "barrage": 19359, + "bethlehem": 19360, + "kahn": 19361, + "##par": 19362, + "mounts": 19363, + "nippon": 19364, + "barony": 19365, + "happier": 19366, + "ryu": 19367, + "makeshift": 19368, + "sheldon": 19369, + "blushed": 19370, + "castillo": 19371, + "barking": 19372, + "listener": 19373, + "taped": 19374, + "bethel": 19375, + "fluent": 19376, + "headlines": 19377, + "pornography": 19378, + "rum": 19379, + "disclosure": 19380, + "sighing": 19381, + "mace": 19382, + "doubling": 19383, + "gunther": 19384, + "manly": 19385, + "##plex": 19386, + "rt": 19387, + "interventions": 19388, + "physiological": 19389, + "forwards": 19390, + "emerges": 19391, + "##tooth": 19392, + "##gny": 19393, + "compliment": 19394, + "rib": 19395, + "recession": 19396, + "visibly": 19397, + "barge": 19398, + "faults": 19399, + "connector": 19400, + "exquisite": 19401, + "prefect": 19402, + "##rlin": 19403, + "patio": 19404, + "##cured": 19405, + "elevators": 19406, + "brandt": 19407, + "italics": 19408, + "pena": 19409, + "173": 19410, + "wasp": 19411, + "satin": 19412, + "ea": 19413, + "botswana": 19414, + "graceful": 19415, + "respectable": 19416, + "##jima": 19417, + "##rter": 19418, + "##oic": 19419, + "franciscan": 19420, + "generates": 19421, + "##dl": 19422, + "alfredo": 19423, + "disgusting": 19424, + "##olate": 19425, + "##iously": 19426, + "sherwood": 19427, + "warns": 19428, + "cod": 19429, + "promo": 19430, + "cheryl": 19431, + "sino": 19432, + "##ة": 19433, + "##escu": 19434, + "twitch": 19435, + "##zhi": 19436, + "brownish": 19437, + "thom": 19438, + "ortiz": 19439, + "##dron": 19440, + "densely": 19441, + "##beat": 19442, + "carmel": 19443, + "reinforce": 19444, + "##bana": 19445, + "187": 19446, + "anastasia": 19447, + "downhill": 19448, + "vertex": 19449, + "contaminated": 19450, + "remembrance": 19451, + "harmonic": 19452, + "homework": 19453, + "##sol": 19454, + "fiancee": 19455, + "gears": 19456, + "olds": 19457, + "angelica": 19458, + "loft": 19459, + "ramsay": 19460, + "quiz": 19461, + "colliery": 19462, + "sevens": 19463, + "##cape": 19464, + "autism": 19465, + "##hil": 19466, + "walkway": 19467, + "##boats": 19468, + "ruben": 19469, + "abnormal": 19470, + "ounce": 19471, + "khmer": 19472, + "##bbe": 19473, + "zachary": 19474, + "bedside": 19475, + "morphology": 19476, + "punching": 19477, + "##olar": 19478, + "sparrow": 19479, + "convinces": 19480, + "##35": 19481, + "hewitt": 19482, + "queer": 19483, + "remastered": 19484, + "rods": 19485, + "mabel": 19486, + "solemn": 19487, + "notified": 19488, + "lyricist": 19489, + "symmetric": 19490, + "##xide": 19491, + "174": 19492, + "encore": 19493, + "passports": 19494, + "wildcats": 19495, + "##uni": 19496, + "baja": 19497, + "##pac": 19498, + "mildly": 19499, + "##ease": 19500, + "bleed": 19501, + "commodity": 19502, + "mounds": 19503, + "glossy": 19504, + "orchestras": 19505, + "##omo": 19506, + "damian": 19507, + "prelude": 19508, + "ambitions": 19509, + "##vet": 19510, + "awhile": 19511, + "remotely": 19512, + "##aud": 19513, + "asserts": 19514, + "imply": 19515, + "##iques": 19516, + "distinctly": 19517, + "modelling": 19518, + "remedy": 19519, + "##dded": 19520, + "windshield": 19521, + "dani": 19522, + "xiao": 19523, + "##endra": 19524, + "audible": 19525, + "powerplant": 19526, + "1300": 19527, + "invalid": 19528, + "elemental": 19529, + "acquisitions": 19530, + "##hala": 19531, + "immaculate": 19532, + "libby": 19533, + "plata": 19534, + "smuggling": 19535, + "ventilation": 19536, + "denoted": 19537, + "minh": 19538, + "##morphism": 19539, + "430": 19540, + "differed": 19541, + "dion": 19542, + "kelley": 19543, + "lore": 19544, + "mocking": 19545, + "sabbath": 19546, + "spikes": 19547, + "hygiene": 19548, + "drown": 19549, + "runoff": 19550, + "stylized": 19551, + "tally": 19552, + "liberated": 19553, + "aux": 19554, + "interpreter": 19555, + "righteous": 19556, + "aba": 19557, + "siren": 19558, + "reaper": 19559, + "pearce": 19560, + "millie": 19561, + "##cier": 19562, + "##yra": 19563, + "gaius": 19564, + "##iso": 19565, + "captures": 19566, + "##ttering": 19567, + "dorm": 19568, + "claudio": 19569, + "##sic": 19570, + "benches": 19571, + "knighted": 19572, + "blackness": 19573, + "##ored": 19574, + "discount": 19575, + "fumble": 19576, + "oxidation": 19577, + "routed": 19578, + "##ς": 19579, + "novak": 19580, + "perpendicular": 19581, + "spoiled": 19582, + "fracture": 19583, + "splits": 19584, + "##urt": 19585, + "pads": 19586, + "topology": 19587, + "##cats": 19588, + "axes": 19589, + "fortunate": 19590, + "offenders": 19591, + "protestants": 19592, + "esteem": 19593, + "221": 19594, + "broadband": 19595, + "convened": 19596, + "frankly": 19597, + "hound": 19598, + "prototypes": 19599, + "isil": 19600, + "facilitated": 19601, + "keel": 19602, + "##sher": 19603, + "sahara": 19604, + "awaited": 19605, + "bubba": 19606, + "orb": 19607, + "prosecutors": 19608, + "186": 19609, + "hem": 19610, + "520": 19611, + "##xing": 19612, + "relaxing": 19613, + "remnant": 19614, + "romney": 19615, + "sorted": 19616, + "slalom": 19617, + "stefano": 19618, + "ulrich": 19619, + "##active": 19620, + "exemption": 19621, + "folder": 19622, + "pauses": 19623, + "foliage": 19624, + "hitchcock": 19625, + "epithet": 19626, + "204": 19627, + "criticisms": 19628, + "##aca": 19629, + "ballistic": 19630, + "brody": 19631, + "hinduism": 19632, + "chaotic": 19633, + "youths": 19634, + "equals": 19635, + "##pala": 19636, + "pts": 19637, + "thicker": 19638, + "analogous": 19639, + "capitalist": 19640, + "improvised": 19641, + "overseeing": 19642, + "sinatra": 19643, + "ascended": 19644, + "beverage": 19645, + "##tl": 19646, + "straightforward": 19647, + "##kon": 19648, + "curran": 19649, + "##west": 19650, + "bois": 19651, + "325": 19652, + "induce": 19653, + "surveying": 19654, + "emperors": 19655, + "sax": 19656, + "unpopular": 19657, + "##kk": 19658, + "cartoonist": 19659, + "fused": 19660, + "##mble": 19661, + "unto": 19662, + "##yuki": 19663, + "localities": 19664, + "##cko": 19665, + "##ln": 19666, + "darlington": 19667, + "slain": 19668, + "academie": 19669, + "lobbying": 19670, + "sediment": 19671, + "puzzles": 19672, + "##grass": 19673, + "defiance": 19674, + "dickens": 19675, + "manifest": 19676, + "tongues": 19677, + "alumnus": 19678, + "arbor": 19679, + "coincide": 19680, + "184": 19681, + "appalachian": 19682, + "mustafa": 19683, + "examiner": 19684, + "cabaret": 19685, + "traumatic": 19686, + "yves": 19687, + "bracelet": 19688, + "draining": 19689, + "heroin": 19690, + "magnum": 19691, + "baths": 19692, + "odessa": 19693, + "consonants": 19694, + "mitsubishi": 19695, + "##gua": 19696, + "kellan": 19697, + "vaudeville": 19698, + "##fr": 19699, + "joked": 19700, + "null": 19701, + "straps": 19702, + "probation": 19703, + "##ław": 19704, + "ceded": 19705, + "interfaces": 19706, + "##pas": 19707, + "##zawa": 19708, + "blinding": 19709, + "viet": 19710, + "224": 19711, + "rothschild": 19712, + "museo": 19713, + "640": 19714, + "huddersfield": 19715, + "##vr": 19716, + "tactic": 19717, + "##storm": 19718, + "brackets": 19719, + "dazed": 19720, + "incorrectly": 19721, + "##vu": 19722, + "reg": 19723, + "glazed": 19724, + "fearful": 19725, + "manifold": 19726, + "benefited": 19727, + "irony": 19728, + "##sun": 19729, + "stumbling": 19730, + "##rte": 19731, + "willingness": 19732, + "balkans": 19733, + "mei": 19734, + "wraps": 19735, + "##aba": 19736, + "injected": 19737, + "##lea": 19738, + "gu": 19739, + "syed": 19740, + "harmless": 19741, + "##hammer": 19742, + "bray": 19743, + "takeoff": 19744, + "poppy": 19745, + "timor": 19746, + "cardboard": 19747, + "astronaut": 19748, + "purdue": 19749, + "weeping": 19750, + "southbound": 19751, + "cursing": 19752, + "stalls": 19753, + "diagonal": 19754, + "##neer": 19755, + "lamar": 19756, + "bryce": 19757, + "comte": 19758, + "weekdays": 19759, + "harrington": 19760, + "##uba": 19761, + "negatively": 19762, + "##see": 19763, + "lays": 19764, + "grouping": 19765, + "##cken": 19766, + "##henko": 19767, + "affirmed": 19768, + "halle": 19769, + "modernist": 19770, + "##lai": 19771, + "hodges": 19772, + "smelling": 19773, + "aristocratic": 19774, + "baptized": 19775, + "dismiss": 19776, + "justification": 19777, + "oilers": 19778, + "##now": 19779, + "coupling": 19780, + "qin": 19781, + "snack": 19782, + "healer": 19783, + "##qing": 19784, + "gardener": 19785, + "layla": 19786, + "battled": 19787, + "formulated": 19788, + "stephenson": 19789, + "gravitational": 19790, + "##gill": 19791, + "##jun": 19792, + "1768": 19793, + "granny": 19794, + "coordinating": 19795, + "suites": 19796, + "##cd": 19797, + "##ioned": 19798, + "monarchs": 19799, + "##cote": 19800, + "##hips": 19801, + "sep": 19802, + "blended": 19803, + "apr": 19804, + "barrister": 19805, + "deposition": 19806, + "fia": 19807, + "mina": 19808, + "policemen": 19809, + "paranoid": 19810, + "##pressed": 19811, + "churchyard": 19812, + "covert": 19813, + "crumpled": 19814, + "creep": 19815, + "abandoning": 19816, + "tr": 19817, + "transmit": 19818, + "conceal": 19819, + "barr": 19820, + "understands": 19821, + "readiness": 19822, + "spire": 19823, + "##cology": 19824, + "##enia": 19825, + "##erry": 19826, + "610": 19827, + "startling": 19828, + "unlock": 19829, + "vida": 19830, + "bowled": 19831, + "slots": 19832, + "##nat": 19833, + "##islav": 19834, + "spaced": 19835, + "trusting": 19836, + "admire": 19837, + "rig": 19838, + "##ink": 19839, + "slack": 19840, + "##70": 19841, + "mv": 19842, + "207": 19843, + "casualty": 19844, + "##wei": 19845, + "classmates": 19846, + "##odes": 19847, + "##rar": 19848, + "##rked": 19849, + "amherst": 19850, + "furnished": 19851, + "evolve": 19852, + "foundry": 19853, + "menace": 19854, + "mead": 19855, + "##lein": 19856, + "flu": 19857, + "wesleyan": 19858, + "##kled": 19859, + "monterey": 19860, + "webber": 19861, + "##vos": 19862, + "wil": 19863, + "##mith": 19864, + "##на": 19865, + "bartholomew": 19866, + "justices": 19867, + "restrained": 19868, + "##cke": 19869, + "amenities": 19870, + "191": 19871, + "mediated": 19872, + "sewage": 19873, + "trenches": 19874, + "ml": 19875, + "mainz": 19876, + "##thus": 19877, + "1800s": 19878, + "##cula": 19879, + "##inski": 19880, + "caine": 19881, + "bonding": 19882, + "213": 19883, + "converts": 19884, + "spheres": 19885, + "superseded": 19886, + "marianne": 19887, + "crypt": 19888, + "sweaty": 19889, + "ensign": 19890, + "historia": 19891, + "##br": 19892, + "spruce": 19893, + "##post": 19894, + "##ask": 19895, + "forks": 19896, + "thoughtfully": 19897, + "yukon": 19898, + "pamphlet": 19899, + "ames": 19900, + "##uter": 19901, + "karma": 19902, + "##yya": 19903, + "bryn": 19904, + "negotiation": 19905, + "sighs": 19906, + "incapable": 19907, + "##mbre": 19908, + "##ntial": 19909, + "actresses": 19910, + "taft": 19911, + "##mill": 19912, + "luce": 19913, + "prevailed": 19914, + "##amine": 19915, + "1773": 19916, + "motionless": 19917, + "envoy": 19918, + "testify": 19919, + "investing": 19920, + "sculpted": 19921, + "instructors": 19922, + "provence": 19923, + "kali": 19924, + "cullen": 19925, + "horseback": 19926, + "##while": 19927, + "goodwin": 19928, + "##jos": 19929, + "gaa": 19930, + "norte": 19931, + "##ldon": 19932, + "modify": 19933, + "wavelength": 19934, + "abd": 19935, + "214": 19936, + "skinned": 19937, + "sprinter": 19938, + "forecast": 19939, + "scheduling": 19940, + "marries": 19941, + "squared": 19942, + "tentative": 19943, + "##chman": 19944, + "boer": 19945, + "##isch": 19946, + "bolts": 19947, + "swap": 19948, + "fisherman": 19949, + "assyrian": 19950, + "impatiently": 19951, + "guthrie": 19952, + "martins": 19953, + "murdoch": 19954, + "194": 19955, + "tanya": 19956, + "nicely": 19957, + "dolly": 19958, + "lacy": 19959, + "med": 19960, + "##45": 19961, + "syn": 19962, + "decks": 19963, + "fashionable": 19964, + "millionaire": 19965, + "##ust": 19966, + "surfing": 19967, + "##ml": 19968, + "##ision": 19969, + "heaved": 19970, + "tammy": 19971, + "consulate": 19972, + "attendees": 19973, + "routinely": 19974, + "197": 19975, + "fuse": 19976, + "saxophonist": 19977, + "backseat": 19978, + "malaya": 19979, + "##lord": 19980, + "scowl": 19981, + "tau": 19982, + "##ishly": 19983, + "193": 19984, + "sighted": 19985, + "steaming": 19986, + "##rks": 19987, + "303": 19988, + "911": 19989, + "##holes": 19990, + "##hong": 19991, + "ching": 19992, + "##wife": 19993, + "bless": 19994, + "conserved": 19995, + "jurassic": 19996, + "stacey": 19997, + "unix": 19998, + "zion": 19999, + "chunk": 20000, + "rigorous": 20001, + "blaine": 20002, + "198": 20003, + "peabody": 20004, + "slayer": 20005, + "dismay": 20006, + "brewers": 20007, + "nz": 20008, + "##jer": 20009, + "det": 20010, + "##glia": 20011, + "glover": 20012, + "postwar": 20013, + "int": 20014, + "penetration": 20015, + "sylvester": 20016, + "imitation": 20017, + "vertically": 20018, + "airlift": 20019, + "heiress": 20020, + "knoxville": 20021, + "viva": 20022, + "##uin": 20023, + "390": 20024, + "macon": 20025, + "##rim": 20026, + "##fighter": 20027, + "##gonal": 20028, + "janice": 20029, + "##orescence": 20030, + "##wari": 20031, + "marius": 20032, + "belongings": 20033, + "leicestershire": 20034, + "196": 20035, + "blanco": 20036, + "inverted": 20037, + "preseason": 20038, + "sanity": 20039, + "sobbing": 20040, + "##due": 20041, + "##elt": 20042, + "##dled": 20043, + "collingwood": 20044, + "regeneration": 20045, + "flickering": 20046, + "shortest": 20047, + "##mount": 20048, + "##osi": 20049, + "feminism": 20050, + "##lat": 20051, + "sherlock": 20052, + "cabinets": 20053, + "fumbled": 20054, + "northbound": 20055, + "precedent": 20056, + "snaps": 20057, + "##mme": 20058, + "researching": 20059, + "##akes": 20060, + "guillaume": 20061, + "insights": 20062, + "manipulated": 20063, + "vapor": 20064, + "neighbour": 20065, + "sap": 20066, + "gangster": 20067, + "frey": 20068, + "f1": 20069, + "stalking": 20070, + "scarcely": 20071, + "callie": 20072, + "barnett": 20073, + "tendencies": 20074, + "audi": 20075, + "doomed": 20076, + "assessing": 20077, + "slung": 20078, + "panchayat": 20079, + "ambiguous": 20080, + "bartlett": 20081, + "##etto": 20082, + "distributing": 20083, + "violating": 20084, + "wolverhampton": 20085, + "##hetic": 20086, + "swami": 20087, + "histoire": 20088, + "##urus": 20089, + "liable": 20090, + "pounder": 20091, + "groin": 20092, + "hussain": 20093, + "larsen": 20094, + "popping": 20095, + "surprises": 20096, + "##atter": 20097, + "vie": 20098, + "curt": 20099, + "##station": 20100, + "mute": 20101, + "relocate": 20102, + "musicals": 20103, + "authorization": 20104, + "richter": 20105, + "##sef": 20106, + "immortality": 20107, + "tna": 20108, + "bombings": 20109, + "##press": 20110, + "deteriorated": 20111, + "yiddish": 20112, + "##acious": 20113, + "robbed": 20114, + "colchester": 20115, + "cs": 20116, + "pmid": 20117, + "ao": 20118, + "verified": 20119, + "balancing": 20120, + "apostle": 20121, + "swayed": 20122, + "recognizable": 20123, + "oxfordshire": 20124, + "retention": 20125, + "nottinghamshire": 20126, + "contender": 20127, + "judd": 20128, + "invitational": 20129, + "shrimp": 20130, + "uhf": 20131, + "##icient": 20132, + "cleaner": 20133, + "longitudinal": 20134, + "tanker": 20135, + "##mur": 20136, + "acronym": 20137, + "broker": 20138, + "koppen": 20139, + "sundance": 20140, + "suppliers": 20141, + "##gil": 20142, + "4000": 20143, + "clipped": 20144, + "fuels": 20145, + "petite": 20146, + "##anne": 20147, + "landslide": 20148, + "helene": 20149, + "diversion": 20150, + "populous": 20151, + "landowners": 20152, + "auspices": 20153, + "melville": 20154, + "quantitative": 20155, + "##xes": 20156, + "ferries": 20157, + "nicky": 20158, + "##llus": 20159, + "doo": 20160, + "haunting": 20161, + "roche": 20162, + "carver": 20163, + "downed": 20164, + "unavailable": 20165, + "##pathy": 20166, + "approximation": 20167, + "hiroshima": 20168, + "##hue": 20169, + "garfield": 20170, + "valle": 20171, + "comparatively": 20172, + "keyboardist": 20173, + "traveler": 20174, + "##eit": 20175, + "congestion": 20176, + "calculating": 20177, + "subsidiaries": 20178, + "##bate": 20179, + "serb": 20180, + "modernization": 20181, + "fairies": 20182, + "deepened": 20183, + "ville": 20184, + "averages": 20185, + "##lore": 20186, + "inflammatory": 20187, + "tonga": 20188, + "##itch": 20189, + "co₂": 20190, + "squads": 20191, + "##hea": 20192, + "gigantic": 20193, + "serum": 20194, + "enjoyment": 20195, + "retailer": 20196, + "verona": 20197, + "35th": 20198, + "cis": 20199, + "##phobic": 20200, + "magna": 20201, + "technicians": 20202, + "##vati": 20203, + "arithmetic": 20204, + "##sport": 20205, + "levin": 20206, + "##dation": 20207, + "amtrak": 20208, + "chow": 20209, + "sienna": 20210, + "##eyer": 20211, + "backstage": 20212, + "entrepreneurship": 20213, + "##otic": 20214, + "learnt": 20215, + "tao": 20216, + "##udy": 20217, + "worcestershire": 20218, + "formulation": 20219, + "baggage": 20220, + "hesitant": 20221, + "bali": 20222, + "sabotage": 20223, + "##kari": 20224, + "barren": 20225, + "enhancing": 20226, + "murmur": 20227, + "pl": 20228, + "freshly": 20229, + "putnam": 20230, + "syntax": 20231, + "aces": 20232, + "medicines": 20233, + "resentment": 20234, + "bandwidth": 20235, + "##sier": 20236, + "grins": 20237, + "chili": 20238, + "guido": 20239, + "##sei": 20240, + "framing": 20241, + "implying": 20242, + "gareth": 20243, + "lissa": 20244, + "genevieve": 20245, + "pertaining": 20246, + "admissions": 20247, + "geo": 20248, + "thorpe": 20249, + "proliferation": 20250, + "sato": 20251, + "bela": 20252, + "analyzing": 20253, + "parting": 20254, + "##gor": 20255, + "awakened": 20256, + "##isman": 20257, + "huddled": 20258, + "secrecy": 20259, + "##kling": 20260, + "hush": 20261, + "gentry": 20262, + "540": 20263, + "dungeons": 20264, + "##ego": 20265, + "coasts": 20266, + "##utz": 20267, + "sacrificed": 20268, + "##chule": 20269, + "landowner": 20270, + "mutually": 20271, + "prevalence": 20272, + "programmer": 20273, + "adolescent": 20274, + "disrupted": 20275, + "seaside": 20276, + "gee": 20277, + "trusts": 20278, + "vamp": 20279, + "georgie": 20280, + "##nesian": 20281, + "##iol": 20282, + "schedules": 20283, + "sindh": 20284, + "##market": 20285, + "etched": 20286, + "hm": 20287, + "sparse": 20288, + "bey": 20289, + "beaux": 20290, + "scratching": 20291, + "gliding": 20292, + "unidentified": 20293, + "216": 20294, + "collaborating": 20295, + "gems": 20296, + "jesuits": 20297, + "oro": 20298, + "accumulation": 20299, + "shaping": 20300, + "mbe": 20301, + "anal": 20302, + "##xin": 20303, + "231": 20304, + "enthusiasts": 20305, + "newscast": 20306, + "##egan": 20307, + "janata": 20308, + "dewey": 20309, + "parkinson": 20310, + "179": 20311, + "ankara": 20312, + "biennial": 20313, + "towering": 20314, + "dd": 20315, + "inconsistent": 20316, + "950": 20317, + "##chet": 20318, + "thriving": 20319, + "terminate": 20320, + "cabins": 20321, + "furiously": 20322, + "eats": 20323, + "advocating": 20324, + "donkey": 20325, + "marley": 20326, + "muster": 20327, + "phyllis": 20328, + "leiden": 20329, + "##user": 20330, + "grassland": 20331, + "glittering": 20332, + "iucn": 20333, + "loneliness": 20334, + "217": 20335, + "memorandum": 20336, + "armenians": 20337, + "##ddle": 20338, + "popularized": 20339, + "rhodesia": 20340, + "60s": 20341, + "lame": 20342, + "##illon": 20343, + "sans": 20344, + "bikini": 20345, + "header": 20346, + "orbits": 20347, + "##xx": 20348, + "##finger": 20349, + "##ulator": 20350, + "sharif": 20351, + "spines": 20352, + "biotechnology": 20353, + "strolled": 20354, + "naughty": 20355, + "yates": 20356, + "##wire": 20357, + "fremantle": 20358, + "milo": 20359, + "##mour": 20360, + "abducted": 20361, + "removes": 20362, + "##atin": 20363, + "humming": 20364, + "wonderland": 20365, + "##chrome": 20366, + "##ester": 20367, + "hume": 20368, + "pivotal": 20369, + "##rates": 20370, + "armand": 20371, + "grams": 20372, + "believers": 20373, + "elector": 20374, + "rte": 20375, + "apron": 20376, + "bis": 20377, + "scraped": 20378, + "##yria": 20379, + "endorsement": 20380, + "initials": 20381, + "##llation": 20382, + "eps": 20383, + "dotted": 20384, + "hints": 20385, + "buzzing": 20386, + "emigration": 20387, + "nearer": 20388, + "##tom": 20389, + "indicators": 20390, + "##ulu": 20391, + "coarse": 20392, + "neutron": 20393, + "protectorate": 20394, + "##uze": 20395, + "directional": 20396, + "exploits": 20397, + "pains": 20398, + "loire": 20399, + "1830s": 20400, + "proponents": 20401, + "guggenheim": 20402, + "rabbits": 20403, + "ritchie": 20404, + "305": 20405, + "hectare": 20406, + "inputs": 20407, + "hutton": 20408, + "##raz": 20409, + "verify": 20410, + "##ako": 20411, + "boilers": 20412, + "longitude": 20413, + "##lev": 20414, + "skeletal": 20415, + "yer": 20416, + "emilia": 20417, + "citrus": 20418, + "compromised": 20419, + "##gau": 20420, + "pokemon": 20421, + "prescription": 20422, + "paragraph": 20423, + "eduard": 20424, + "cadillac": 20425, + "attire": 20426, + "categorized": 20427, + "kenyan": 20428, + "weddings": 20429, + "charley": 20430, + "##bourg": 20431, + "entertain": 20432, + "monmouth": 20433, + "##lles": 20434, + "nutrients": 20435, + "davey": 20436, + "mesh": 20437, + "incentive": 20438, + "practised": 20439, + "ecosystems": 20440, + "kemp": 20441, + "subdued": 20442, + "overheard": 20443, + "##rya": 20444, + "bodily": 20445, + "maxim": 20446, + "##nius": 20447, + "apprenticeship": 20448, + "ursula": 20449, + "##fight": 20450, + "lodged": 20451, + "rug": 20452, + "silesian": 20453, + "unconstitutional": 20454, + "patel": 20455, + "inspected": 20456, + "coyote": 20457, + "unbeaten": 20458, + "##hak": 20459, + "34th": 20460, + "disruption": 20461, + "convict": 20462, + "parcel": 20463, + "##cl": 20464, + "##nham": 20465, + "collier": 20466, + "implicated": 20467, + "mallory": 20468, + "##iac": 20469, + "##lab": 20470, + "susannah": 20471, + "winkler": 20472, + "##rber": 20473, + "shia": 20474, + "phelps": 20475, + "sediments": 20476, + "graphical": 20477, + "robotic": 20478, + "##sner": 20479, + "adulthood": 20480, + "mart": 20481, + "smoked": 20482, + "##isto": 20483, + "kathryn": 20484, + "clarified": 20485, + "##aran": 20486, + "divides": 20487, + "convictions": 20488, + "oppression": 20489, + "pausing": 20490, + "burying": 20491, + "##mt": 20492, + "federico": 20493, + "mathias": 20494, + "eileen": 20495, + "##tana": 20496, + "kite": 20497, + "hunched": 20498, + "##acies": 20499, + "189": 20500, + "##atz": 20501, + "disadvantage": 20502, + "liza": 20503, + "kinetic": 20504, + "greedy": 20505, + "paradox": 20506, + "yokohama": 20507, + "dowager": 20508, + "trunks": 20509, + "ventured": 20510, + "##gement": 20511, + "gupta": 20512, + "vilnius": 20513, + "olaf": 20514, + "##thest": 20515, + "crimean": 20516, + "hopper": 20517, + "##ej": 20518, + "progressively": 20519, + "arturo": 20520, + "mouthed": 20521, + "arrondissement": 20522, + "##fusion": 20523, + "rubin": 20524, + "simulcast": 20525, + "oceania": 20526, + "##orum": 20527, + "##stra": 20528, + "##rred": 20529, + "busiest": 20530, + "intensely": 20531, + "navigator": 20532, + "cary": 20533, + "##vine": 20534, + "##hini": 20535, + "##bies": 20536, + "fife": 20537, + "rowe": 20538, + "rowland": 20539, + "posing": 20540, + "insurgents": 20541, + "shafts": 20542, + "lawsuits": 20543, + "activate": 20544, + "conor": 20545, + "inward": 20546, + "culturally": 20547, + "garlic": 20548, + "265": 20549, + "##eering": 20550, + "eclectic": 20551, + "##hui": 20552, + "##kee": 20553, + "##nl": 20554, + "furrowed": 20555, + "vargas": 20556, + "meteorological": 20557, + "rendezvous": 20558, + "##aus": 20559, + "culinary": 20560, + "commencement": 20561, + "##dition": 20562, + "quota": 20563, + "##notes": 20564, + "mommy": 20565, + "salaries": 20566, + "overlapping": 20567, + "mule": 20568, + "##iology": 20569, + "##mology": 20570, + "sums": 20571, + "wentworth": 20572, + "##isk": 20573, + "##zione": 20574, + "mainline": 20575, + "subgroup": 20576, + "##illy": 20577, + "hack": 20578, + "plaintiff": 20579, + "verdi": 20580, + "bulb": 20581, + "differentiation": 20582, + "engagements": 20583, + "multinational": 20584, + "supplemented": 20585, + "bertrand": 20586, + "caller": 20587, + "regis": 20588, + "##naire": 20589, + "##sler": 20590, + "##arts": 20591, + "##imated": 20592, + "blossom": 20593, + "propagation": 20594, + "kilometer": 20595, + "viaduct": 20596, + "vineyards": 20597, + "##uate": 20598, + "beckett": 20599, + "optimization": 20600, + "golfer": 20601, + "songwriters": 20602, + "seminal": 20603, + "semitic": 20604, + "thud": 20605, + "volatile": 20606, + "evolving": 20607, + "ridley": 20608, + "##wley": 20609, + "trivial": 20610, + "distributions": 20611, + "scandinavia": 20612, + "jiang": 20613, + "##ject": 20614, + "wrestled": 20615, + "insistence": 20616, + "##dio": 20617, + "emphasizes": 20618, + "napkin": 20619, + "##ods": 20620, + "adjunct": 20621, + "rhyme": 20622, + "##ricted": 20623, + "##eti": 20624, + "hopeless": 20625, + "surrounds": 20626, + "tremble": 20627, + "32nd": 20628, + "smoky": 20629, + "##ntly": 20630, + "oils": 20631, + "medicinal": 20632, + "padded": 20633, + "steer": 20634, + "wilkes": 20635, + "219": 20636, + "255": 20637, + "concessions": 20638, + "hue": 20639, + "uniquely": 20640, + "blinded": 20641, + "landon": 20642, + "yahoo": 20643, + "##lane": 20644, + "hendrix": 20645, + "commemorating": 20646, + "dex": 20647, + "specify": 20648, + "chicks": 20649, + "##ggio": 20650, + "intercity": 20651, + "1400": 20652, + "morley": 20653, + "##torm": 20654, + "highlighting": 20655, + "##oting": 20656, + "pang": 20657, + "oblique": 20658, + "stalled": 20659, + "##liner": 20660, + "flirting": 20661, + "newborn": 20662, + "1769": 20663, + "bishopric": 20664, + "shaved": 20665, + "232": 20666, + "currie": 20667, + "##ush": 20668, + "dharma": 20669, + "spartan": 20670, + "##ooped": 20671, + "favorites": 20672, + "smug": 20673, + "novella": 20674, + "sirens": 20675, + "abusive": 20676, + "creations": 20677, + "espana": 20678, + "##lage": 20679, + "paradigm": 20680, + "semiconductor": 20681, + "sheen": 20682, + "##rdo": 20683, + "##yen": 20684, + "##zak": 20685, + "nrl": 20686, + "renew": 20687, + "##pose": 20688, + "##tur": 20689, + "adjutant": 20690, + "marches": 20691, + "norma": 20692, + "##enity": 20693, + "ineffective": 20694, + "weimar": 20695, + "grunt": 20696, + "##gat": 20697, + "lordship": 20698, + "plotting": 20699, + "expenditure": 20700, + "infringement": 20701, + "lbs": 20702, + "refrain": 20703, + "av": 20704, + "mimi": 20705, + "mistakenly": 20706, + "postmaster": 20707, + "1771": 20708, + "##bara": 20709, + "ras": 20710, + "motorsports": 20711, + "tito": 20712, + "199": 20713, + "subjective": 20714, + "##zza": 20715, + "bully": 20716, + "stew": 20717, + "##kaya": 20718, + "prescott": 20719, + "1a": 20720, + "##raphic": 20721, + "##zam": 20722, + "bids": 20723, + "styling": 20724, + "paranormal": 20725, + "reeve": 20726, + "sneaking": 20727, + "exploding": 20728, + "katz": 20729, + "akbar": 20730, + "migrant": 20731, + "syllables": 20732, + "indefinitely": 20733, + "##ogical": 20734, + "destroys": 20735, + "replaces": 20736, + "applause": 20737, + "##phine": 20738, + "pest": 20739, + "##fide": 20740, + "218": 20741, + "articulated": 20742, + "bertie": 20743, + "##thing": 20744, + "##cars": 20745, + "##ptic": 20746, + "courtroom": 20747, + "crowley": 20748, + "aesthetics": 20749, + "cummings": 20750, + "tehsil": 20751, + "hormones": 20752, + "titanic": 20753, + "dangerously": 20754, + "##ibe": 20755, + "stadion": 20756, + "jaenelle": 20757, + "auguste": 20758, + "ciudad": 20759, + "##chu": 20760, + "mysore": 20761, + "partisans": 20762, + "##sio": 20763, + "lucan": 20764, + "philipp": 20765, + "##aly": 20766, + "debating": 20767, + "henley": 20768, + "interiors": 20769, + "##rano": 20770, + "##tious": 20771, + "homecoming": 20772, + "beyonce": 20773, + "usher": 20774, + "henrietta": 20775, + "prepares": 20776, + "weeds": 20777, + "##oman": 20778, + "ely": 20779, + "plucked": 20780, + "##pire": 20781, + "##dable": 20782, + "luxurious": 20783, + "##aq": 20784, + "artifact": 20785, + "password": 20786, + "pasture": 20787, + "juno": 20788, + "maddy": 20789, + "minsk": 20790, + "##dder": 20791, + "##ologies": 20792, + "##rone": 20793, + "assessments": 20794, + "martian": 20795, + "royalist": 20796, + "1765": 20797, + "examines": 20798, + "##mani": 20799, + "##rge": 20800, + "nino": 20801, + "223": 20802, + "parry": 20803, + "scooped": 20804, + "relativity": 20805, + "##eli": 20806, + "##uting": 20807, + "##cao": 20808, + "congregational": 20809, + "noisy": 20810, + "traverse": 20811, + "##agawa": 20812, + "strikeouts": 20813, + "nickelodeon": 20814, + "obituary": 20815, + "transylvania": 20816, + "binds": 20817, + "depictions": 20818, + "polk": 20819, + "trolley": 20820, + "##yed": 20821, + "##lard": 20822, + "breeders": 20823, + "##under": 20824, + "dryly": 20825, + "hokkaido": 20826, + "1762": 20827, + "strengths": 20828, + "stacks": 20829, + "bonaparte": 20830, + "connectivity": 20831, + "neared": 20832, + "prostitutes": 20833, + "stamped": 20834, + "anaheim": 20835, + "gutierrez": 20836, + "sinai": 20837, + "##zzling": 20838, + "bram": 20839, + "fresno": 20840, + "madhya": 20841, + "##86": 20842, + "proton": 20843, + "##lena": 20844, + "##llum": 20845, + "##phon": 20846, + "reelected": 20847, + "wanda": 20848, + "##anus": 20849, + "##lb": 20850, + "ample": 20851, + "distinguishing": 20852, + "##yler": 20853, + "grasping": 20854, + "sermons": 20855, + "tomato": 20856, + "bland": 20857, + "stimulation": 20858, + "avenues": 20859, + "##eux": 20860, + "spreads": 20861, + "scarlett": 20862, + "fern": 20863, + "pentagon": 20864, + "assert": 20865, + "baird": 20866, + "chesapeake": 20867, + "ir": 20868, + "calmed": 20869, + "distortion": 20870, + "fatalities": 20871, + "##olis": 20872, + "correctional": 20873, + "pricing": 20874, + "##astic": 20875, + "##gina": 20876, + "prom": 20877, + "dammit": 20878, + "ying": 20879, + "collaborate": 20880, + "##chia": 20881, + "welterweight": 20882, + "33rd": 20883, + "pointer": 20884, + "substitution": 20885, + "bonded": 20886, + "umpire": 20887, + "communicating": 20888, + "multitude": 20889, + "paddle": 20890, + "##obe": 20891, + "federally": 20892, + "intimacy": 20893, + "##insky": 20894, + "betray": 20895, + "ssr": 20896, + "##lett": 20897, + "##lean": 20898, + "##lves": 20899, + "##therapy": 20900, + "airbus": 20901, + "##tery": 20902, + "functioned": 20903, + "ud": 20904, + "bearer": 20905, + "biomedical": 20906, + "netflix": 20907, + "##hire": 20908, + "##nca": 20909, + "condom": 20910, + "brink": 20911, + "ik": 20912, + "##nical": 20913, + "macy": 20914, + "##bet": 20915, + "flap": 20916, + "gma": 20917, + "experimented": 20918, + "jelly": 20919, + "lavender": 20920, + "##icles": 20921, + "##ulia": 20922, + "munro": 20923, + "##mian": 20924, + "##tial": 20925, + "rye": 20926, + "##rle": 20927, + "60th": 20928, + "gigs": 20929, + "hottest": 20930, + "rotated": 20931, + "predictions": 20932, + "fuji": 20933, + "bu": 20934, + "##erence": 20935, + "##omi": 20936, + "barangay": 20937, + "##fulness": 20938, + "##sas": 20939, + "clocks": 20940, + "##rwood": 20941, + "##liness": 20942, + "cereal": 20943, + "roe": 20944, + "wight": 20945, + "decker": 20946, + "uttered": 20947, + "babu": 20948, + "onion": 20949, + "xml": 20950, + "forcibly": 20951, + "##df": 20952, + "petra": 20953, + "sarcasm": 20954, + "hartley": 20955, + "peeled": 20956, + "storytelling": 20957, + "##42": 20958, + "##xley": 20959, + "##ysis": 20960, + "##ffa": 20961, + "fibre": 20962, + "kiel": 20963, + "auditor": 20964, + "fig": 20965, + "harald": 20966, + "greenville": 20967, + "##berries": 20968, + "geographically": 20969, + "nell": 20970, + "quartz": 20971, + "##athic": 20972, + "cemeteries": 20973, + "##lr": 20974, + "crossings": 20975, + "nah": 20976, + "holloway": 20977, + "reptiles": 20978, + "chun": 20979, + "sichuan": 20980, + "snowy": 20981, + "660": 20982, + "corrections": 20983, + "##ivo": 20984, + "zheng": 20985, + "ambassadors": 20986, + "blacksmith": 20987, + "fielded": 20988, + "fluids": 20989, + "hardcover": 20990, + "turnover": 20991, + "medications": 20992, + "melvin": 20993, + "academies": 20994, + "##erton": 20995, + "ro": 20996, + "roach": 20997, + "absorbing": 20998, + "spaniards": 20999, + "colton": 21000, + "##founded": 21001, + "outsider": 21002, + "espionage": 21003, + "kelsey": 21004, + "245": 21005, + "edible": 21006, + "##ulf": 21007, + "dora": 21008, + "establishes": 21009, + "##sham": 21010, + "##tries": 21011, + "contracting": 21012, + "##tania": 21013, + "cinematic": 21014, + "costello": 21015, + "nesting": 21016, + "##uron": 21017, + "connolly": 21018, + "duff": 21019, + "##nology": 21020, + "mma": 21021, + "##mata": 21022, + "fergus": 21023, + "sexes": 21024, + "gi": 21025, + "optics": 21026, + "spectator": 21027, + "woodstock": 21028, + "banning": 21029, + "##hee": 21030, + "##fle": 21031, + "differentiate": 21032, + "outfielder": 21033, + "refinery": 21034, + "226": 21035, + "312": 21036, + "gerhard": 21037, + "horde": 21038, + "lair": 21039, + "drastically": 21040, + "##udi": 21041, + "landfall": 21042, + "##cheng": 21043, + "motorsport": 21044, + "odi": 21045, + "##achi": 21046, + "predominant": 21047, + "quay": 21048, + "skins": 21049, + "##ental": 21050, + "edna": 21051, + "harshly": 21052, + "complementary": 21053, + "murdering": 21054, + "##aves": 21055, + "wreckage": 21056, + "##90": 21057, + "ono": 21058, + "outstretched": 21059, + "lennox": 21060, + "munitions": 21061, + "galen": 21062, + "reconcile": 21063, + "470": 21064, + "scalp": 21065, + "bicycles": 21066, + "gillespie": 21067, + "questionable": 21068, + "rosenberg": 21069, + "guillermo": 21070, + "hostel": 21071, + "jarvis": 21072, + "kabul": 21073, + "volvo": 21074, + "opium": 21075, + "yd": 21076, + "##twined": 21077, + "abuses": 21078, + "decca": 21079, + "outpost": 21080, + "##cino": 21081, + "sensible": 21082, + "neutrality": 21083, + "##64": 21084, + "ponce": 21085, + "anchorage": 21086, + "atkins": 21087, + "turrets": 21088, + "inadvertently": 21089, + "disagree": 21090, + "libre": 21091, + "vodka": 21092, + "reassuring": 21093, + "weighs": 21094, + "##yal": 21095, + "glide": 21096, + "jumper": 21097, + "ceilings": 21098, + "repertory": 21099, + "outs": 21100, + "stain": 21101, + "##bial": 21102, + "envy": 21103, + "##ucible": 21104, + "smashing": 21105, + "heightened": 21106, + "policing": 21107, + "hyun": 21108, + "mixes": 21109, + "lai": 21110, + "prima": 21111, + "##ples": 21112, + "celeste": 21113, + "##bina": 21114, + "lucrative": 21115, + "intervened": 21116, + "kc": 21117, + "manually": 21118, + "##rned": 21119, + "stature": 21120, + "staffed": 21121, + "bun": 21122, + "bastards": 21123, + "nairobi": 21124, + "priced": 21125, + "##auer": 21126, + "thatcher": 21127, + "##kia": 21128, + "tripped": 21129, + "comune": 21130, + "##ogan": 21131, + "##pled": 21132, + "brasil": 21133, + "incentives": 21134, + "emanuel": 21135, + "hereford": 21136, + "musica": 21137, + "##kim": 21138, + "benedictine": 21139, + "biennale": 21140, + "##lani": 21141, + "eureka": 21142, + "gardiner": 21143, + "rb": 21144, + "knocks": 21145, + "sha": 21146, + "##ael": 21147, + "##elled": 21148, + "##onate": 21149, + "efficacy": 21150, + "ventura": 21151, + "masonic": 21152, + "sanford": 21153, + "maize": 21154, + "leverage": 21155, + "##feit": 21156, + "capacities": 21157, + "santana": 21158, + "##aur": 21159, + "novelty": 21160, + "vanilla": 21161, + "##cter": 21162, + "##tour": 21163, + "benin": 21164, + "##oir": 21165, + "##rain": 21166, + "neptune": 21167, + "drafting": 21168, + "tallinn": 21169, + "##cable": 21170, + "humiliation": 21171, + "##boarding": 21172, + "schleswig": 21173, + "fabian": 21174, + "bernardo": 21175, + "liturgy": 21176, + "spectacle": 21177, + "sweeney": 21178, + "pont": 21179, + "routledge": 21180, + "##tment": 21181, + "cosmos": 21182, + "ut": 21183, + "hilt": 21184, + "sleek": 21185, + "universally": 21186, + "##eville": 21187, + "##gawa": 21188, + "typed": 21189, + "##dry": 21190, + "favors": 21191, + "allegheny": 21192, + "glaciers": 21193, + "##rly": 21194, + "recalling": 21195, + "aziz": 21196, + "##log": 21197, + "parasite": 21198, + "requiem": 21199, + "auf": 21200, + "##berto": 21201, + "##llin": 21202, + "illumination": 21203, + "##breaker": 21204, + "##issa": 21205, + "festivities": 21206, + "bows": 21207, + "govern": 21208, + "vibe": 21209, + "vp": 21210, + "333": 21211, + "sprawled": 21212, + "larson": 21213, + "pilgrim": 21214, + "bwf": 21215, + "leaping": 21216, + "##rts": 21217, + "##ssel": 21218, + "alexei": 21219, + "greyhound": 21220, + "hoarse": 21221, + "##dler": 21222, + "##oration": 21223, + "seneca": 21224, + "##cule": 21225, + "gaping": 21226, + "##ulously": 21227, + "##pura": 21228, + "cinnamon": 21229, + "##gens": 21230, + "##rricular": 21231, + "craven": 21232, + "fantasies": 21233, + "houghton": 21234, + "engined": 21235, + "reigned": 21236, + "dictator": 21237, + "supervising": 21238, + "##oris": 21239, + "bogota": 21240, + "commentaries": 21241, + "unnatural": 21242, + "fingernails": 21243, + "spirituality": 21244, + "tighten": 21245, + "##tm": 21246, + "canadiens": 21247, + "protesting": 21248, + "intentional": 21249, + "cheers": 21250, + "sparta": 21251, + "##ytic": 21252, + "##iere": 21253, + "##zine": 21254, + "widen": 21255, + "belgarath": 21256, + "controllers": 21257, + "dodd": 21258, + "iaaf": 21259, + "navarre": 21260, + "##ication": 21261, + "defect": 21262, + "squire": 21263, + "steiner": 21264, + "whisky": 21265, + "##mins": 21266, + "560": 21267, + "inevitably": 21268, + "tome": 21269, + "##gold": 21270, + "chew": 21271, + "##uid": 21272, + "##lid": 21273, + "elastic": 21274, + "##aby": 21275, + "streaked": 21276, + "alliances": 21277, + "jailed": 21278, + "regal": 21279, + "##ined": 21280, + "##phy": 21281, + "czechoslovak": 21282, + "narration": 21283, + "absently": 21284, + "##uld": 21285, + "bluegrass": 21286, + "guangdong": 21287, + "quran": 21288, + "criticizing": 21289, + "hose": 21290, + "hari": 21291, + "##liest": 21292, + "##owa": 21293, + "skier": 21294, + "streaks": 21295, + "deploy": 21296, + "##lom": 21297, + "raft": 21298, + "bose": 21299, + "dialed": 21300, + "huff": 21301, + "##eira": 21302, + "haifa": 21303, + "simplest": 21304, + "bursting": 21305, + "endings": 21306, + "ib": 21307, + "sultanate": 21308, + "##titled": 21309, + "franks": 21310, + "whitman": 21311, + "ensures": 21312, + "sven": 21313, + "##ggs": 21314, + "collaborators": 21315, + "forster": 21316, + "organising": 21317, + "ui": 21318, + "banished": 21319, + "napier": 21320, + "injustice": 21321, + "teller": 21322, + "layered": 21323, + "thump": 21324, + "##otti": 21325, + "roc": 21326, + "battleships": 21327, + "evidenced": 21328, + "fugitive": 21329, + "sadie": 21330, + "robotics": 21331, + "##roud": 21332, + "equatorial": 21333, + "geologist": 21334, + "##iza": 21335, + "yielding": 21336, + "##bron": 21337, + "##sr": 21338, + "internationale": 21339, + "mecca": 21340, + "##diment": 21341, + "sbs": 21342, + "skyline": 21343, + "toad": 21344, + "uploaded": 21345, + "reflective": 21346, + "undrafted": 21347, + "lal": 21348, + "leafs": 21349, + "bayern": 21350, + "##dai": 21351, + "lakshmi": 21352, + "shortlisted": 21353, + "##stick": 21354, + "##wicz": 21355, + "camouflage": 21356, + "donate": 21357, + "af": 21358, + "christi": 21359, + "lau": 21360, + "##acio": 21361, + "disclosed": 21362, + "nemesis": 21363, + "1761": 21364, + "assemble": 21365, + "straining": 21366, + "northamptonshire": 21367, + "tal": 21368, + "##asi": 21369, + "bernardino": 21370, + "premature": 21371, + "heidi": 21372, + "42nd": 21373, + "coefficients": 21374, + "galactic": 21375, + "reproduce": 21376, + "buzzed": 21377, + "sensations": 21378, + "zionist": 21379, + "monsieur": 21380, + "myrtle": 21381, + "##eme": 21382, + "archery": 21383, + "strangled": 21384, + "musically": 21385, + "viewpoint": 21386, + "antiquities": 21387, + "bei": 21388, + "trailers": 21389, + "seahawks": 21390, + "cured": 21391, + "pee": 21392, + "preferring": 21393, + "tasmanian": 21394, + "lange": 21395, + "sul": 21396, + "##mail": 21397, + "##working": 21398, + "colder": 21399, + "overland": 21400, + "lucivar": 21401, + "massey": 21402, + "gatherings": 21403, + "haitian": 21404, + "##smith": 21405, + "disapproval": 21406, + "flaws": 21407, + "##cco": 21408, + "##enbach": 21409, + "1766": 21410, + "npr": 21411, + "##icular": 21412, + "boroughs": 21413, + "creole": 21414, + "forums": 21415, + "techno": 21416, + "1755": 21417, + "dent": 21418, + "abdominal": 21419, + "streetcar": 21420, + "##eson": 21421, + "##stream": 21422, + "procurement": 21423, + "gemini": 21424, + "predictable": 21425, + "##tya": 21426, + "acheron": 21427, + "christoph": 21428, + "feeder": 21429, + "fronts": 21430, + "vendor": 21431, + "bernhard": 21432, + "jammu": 21433, + "tumors": 21434, + "slang": 21435, + "##uber": 21436, + "goaltender": 21437, + "twists": 21438, + "curving": 21439, + "manson": 21440, + "vuelta": 21441, + "mer": 21442, + "peanut": 21443, + "confessions": 21444, + "pouch": 21445, + "unpredictable": 21446, + "allowance": 21447, + "theodor": 21448, + "vascular": 21449, + "##factory": 21450, + "bala": 21451, + "authenticity": 21452, + "metabolic": 21453, + "coughing": 21454, + "nanjing": 21455, + "##cea": 21456, + "pembroke": 21457, + "##bard": 21458, + "splendid": 21459, + "36th": 21460, + "ff": 21461, + "hourly": 21462, + "##ahu": 21463, + "elmer": 21464, + "handel": 21465, + "##ivate": 21466, + "awarding": 21467, + "thrusting": 21468, + "dl": 21469, + "experimentation": 21470, + "##hesion": 21471, + "##46": 21472, + "caressed": 21473, + "entertained": 21474, + "steak": 21475, + "##rangle": 21476, + "biologist": 21477, + "orphans": 21478, + "baroness": 21479, + "oyster": 21480, + "stepfather": 21481, + "##dridge": 21482, + "mirage": 21483, + "reefs": 21484, + "speeding": 21485, + "##31": 21486, + "barons": 21487, + "1764": 21488, + "227": 21489, + "inhabit": 21490, + "preached": 21491, + "repealed": 21492, + "##tral": 21493, + "honoring": 21494, + "boogie": 21495, + "captives": 21496, + "administer": 21497, + "johanna": 21498, + "##imate": 21499, + "gel": 21500, + "suspiciously": 21501, + "1767": 21502, + "sobs": 21503, + "##dington": 21504, + "backbone": 21505, + "hayward": 21506, + "garry": 21507, + "##folding": 21508, + "##nesia": 21509, + "maxi": 21510, + "##oof": 21511, + "##ppe": 21512, + "ellison": 21513, + "galileo": 21514, + "##stand": 21515, + "crimea": 21516, + "frenzy": 21517, + "amour": 21518, + "bumper": 21519, + "matrices": 21520, + "natalia": 21521, + "baking": 21522, + "garth": 21523, + "palestinians": 21524, + "##grove": 21525, + "smack": 21526, + "conveyed": 21527, + "ensembles": 21528, + "gardening": 21529, + "##manship": 21530, + "##rup": 21531, + "##stituting": 21532, + "1640": 21533, + "harvesting": 21534, + "topography": 21535, + "jing": 21536, + "shifters": 21537, + "dormitory": 21538, + "##carriage": 21539, + "##lston": 21540, + "ist": 21541, + "skulls": 21542, + "##stadt": 21543, + "dolores": 21544, + "jewellery": 21545, + "sarawak": 21546, + "##wai": 21547, + "##zier": 21548, + "fences": 21549, + "christy": 21550, + "confinement": 21551, + "tumbling": 21552, + "credibility": 21553, + "fir": 21554, + "stench": 21555, + "##bria": 21556, + "##plication": 21557, + "##nged": 21558, + "##sam": 21559, + "virtues": 21560, + "##belt": 21561, + "marjorie": 21562, + "pba": 21563, + "##eem": 21564, + "##made": 21565, + "celebrates": 21566, + "schooner": 21567, + "agitated": 21568, + "barley": 21569, + "fulfilling": 21570, + "anthropologist": 21571, + "##pro": 21572, + "restrict": 21573, + "novi": 21574, + "regulating": 21575, + "##nent": 21576, + "padres": 21577, + "##rani": 21578, + "##hesive": 21579, + "loyola": 21580, + "tabitha": 21581, + "milky": 21582, + "olson": 21583, + "proprietor": 21584, + "crambidae": 21585, + "guarantees": 21586, + "intercollegiate": 21587, + "ljubljana": 21588, + "hilda": 21589, + "##sko": 21590, + "ignorant": 21591, + "hooded": 21592, + "##lts": 21593, + "sardinia": 21594, + "##lidae": 21595, + "##vation": 21596, + "frontman": 21597, + "privileged": 21598, + "witchcraft": 21599, + "##gp": 21600, + "jammed": 21601, + "laude": 21602, + "poking": 21603, + "##than": 21604, + "bracket": 21605, + "amazement": 21606, + "yunnan": 21607, + "##erus": 21608, + "maharaja": 21609, + "linnaeus": 21610, + "264": 21611, + "commissioning": 21612, + "milano": 21613, + "peacefully": 21614, + "##logies": 21615, + "akira": 21616, + "rani": 21617, + "regulator": 21618, + "##36": 21619, + "grasses": 21620, + "##rance": 21621, + "luzon": 21622, + "crows": 21623, + "compiler": 21624, + "gretchen": 21625, + "seaman": 21626, + "edouard": 21627, + "tab": 21628, + "buccaneers": 21629, + "ellington": 21630, + "hamlets": 21631, + "whig": 21632, + "socialists": 21633, + "##anto": 21634, + "directorial": 21635, + "easton": 21636, + "mythological": 21637, + "##kr": 21638, + "##vary": 21639, + "rhineland": 21640, + "semantic": 21641, + "taut": 21642, + "dune": 21643, + "inventions": 21644, + "succeeds": 21645, + "##iter": 21646, + "replication": 21647, + "branched": 21648, + "##pired": 21649, + "jul": 21650, + "prosecuted": 21651, + "kangaroo": 21652, + "penetrated": 21653, + "##avian": 21654, + "middlesbrough": 21655, + "doses": 21656, + "bleak": 21657, + "madam": 21658, + "predatory": 21659, + "relentless": 21660, + "##vili": 21661, + "reluctance": 21662, + "##vir": 21663, + "hailey": 21664, + "crore": 21665, + "silvery": 21666, + "1759": 21667, + "monstrous": 21668, + "swimmers": 21669, + "transmissions": 21670, + "hawthorn": 21671, + "informing": 21672, + "##eral": 21673, + "toilets": 21674, + "caracas": 21675, + "crouch": 21676, + "kb": 21677, + "##sett": 21678, + "295": 21679, + "cartel": 21680, + "hadley": 21681, + "##aling": 21682, + "alexia": 21683, + "yvonne": 21684, + "##biology": 21685, + "cinderella": 21686, + "eton": 21687, + "superb": 21688, + "blizzard": 21689, + "stabbing": 21690, + "industrialist": 21691, + "maximus": 21692, + "##gm": 21693, + "##orus": 21694, + "groves": 21695, + "maud": 21696, + "clade": 21697, + "oversized": 21698, + "comedic": 21699, + "##bella": 21700, + "rosen": 21701, + "nomadic": 21702, + "fulham": 21703, + "montane": 21704, + "beverages": 21705, + "galaxies": 21706, + "redundant": 21707, + "swarm": 21708, + "##rot": 21709, + "##folia": 21710, + "##llis": 21711, + "buckinghamshire": 21712, + "fen": 21713, + "bearings": 21714, + "bahadur": 21715, + "##rom": 21716, + "gilles": 21717, + "phased": 21718, + "dynamite": 21719, + "faber": 21720, + "benoit": 21721, + "vip": 21722, + "##ount": 21723, + "##wd": 21724, + "booking": 21725, + "fractured": 21726, + "tailored": 21727, + "anya": 21728, + "spices": 21729, + "westwood": 21730, + "cairns": 21731, + "auditions": 21732, + "inflammation": 21733, + "steamed": 21734, + "##rocity": 21735, + "##acion": 21736, + "##urne": 21737, + "skyla": 21738, + "thereof": 21739, + "watford": 21740, + "torment": 21741, + "archdeacon": 21742, + "transforms": 21743, + "lulu": 21744, + "demeanor": 21745, + "fucked": 21746, + "serge": 21747, + "##sor": 21748, + "mckenna": 21749, + "minas": 21750, + "entertainer": 21751, + "##icide": 21752, + "caress": 21753, + "originate": 21754, + "residue": 21755, + "##sty": 21756, + "1740": 21757, + "##ilised": 21758, + "##org": 21759, + "beech": 21760, + "##wana": 21761, + "subsidies": 21762, + "##ghton": 21763, + "emptied": 21764, + "gladstone": 21765, + "ru": 21766, + "firefighters": 21767, + "voodoo": 21768, + "##rcle": 21769, + "het": 21770, + "nightingale": 21771, + "tamara": 21772, + "edmond": 21773, + "ingredient": 21774, + "weaknesses": 21775, + "silhouette": 21776, + "285": 21777, + "compatibility": 21778, + "withdrawing": 21779, + "hampson": 21780, + "##mona": 21781, + "anguish": 21782, + "giggling": 21783, + "##mber": 21784, + "bookstore": 21785, + "##jiang": 21786, + "southernmost": 21787, + "tilting": 21788, + "##vance": 21789, + "bai": 21790, + "economical": 21791, + "rf": 21792, + "briefcase": 21793, + "dreadful": 21794, + "hinted": 21795, + "projections": 21796, + "shattering": 21797, + "totaling": 21798, + "##rogate": 21799, + "analogue": 21800, + "indicted": 21801, + "periodical": 21802, + "fullback": 21803, + "##dman": 21804, + "haynes": 21805, + "##tenberg": 21806, + "##ffs": 21807, + "##ishment": 21808, + "1745": 21809, + "thirst": 21810, + "stumble": 21811, + "penang": 21812, + "vigorous": 21813, + "##ddling": 21814, + "##kor": 21815, + "##lium": 21816, + "octave": 21817, + "##ove": 21818, + "##enstein": 21819, + "##inen": 21820, + "##ones": 21821, + "siberian": 21822, + "##uti": 21823, + "cbn": 21824, + "repeal": 21825, + "swaying": 21826, + "##vington": 21827, + "khalid": 21828, + "tanaka": 21829, + "unicorn": 21830, + "otago": 21831, + "plastered": 21832, + "lobe": 21833, + "riddle": 21834, + "##rella": 21835, + "perch": 21836, + "##ishing": 21837, + "croydon": 21838, + "filtered": 21839, + "graeme": 21840, + "tripoli": 21841, + "##ossa": 21842, + "crocodile": 21843, + "##chers": 21844, + "sufi": 21845, + "mined": 21846, + "##tung": 21847, + "inferno": 21848, + "lsu": 21849, + "##phi": 21850, + "swelled": 21851, + "utilizes": 21852, + "£2": 21853, + "cale": 21854, + "periodicals": 21855, + "styx": 21856, + "hike": 21857, + "informally": 21858, + "coop": 21859, + "lund": 21860, + "##tidae": 21861, + "ala": 21862, + "hen": 21863, + "qui": 21864, + "transformations": 21865, + "disposed": 21866, + "sheath": 21867, + "chickens": 21868, + "##cade": 21869, + "fitzroy": 21870, + "sas": 21871, + "silesia": 21872, + "unacceptable": 21873, + "odisha": 21874, + "1650": 21875, + "sabrina": 21876, + "pe": 21877, + "spokane": 21878, + "ratios": 21879, + "athena": 21880, + "massage": 21881, + "shen": 21882, + "dilemma": 21883, + "##drum": 21884, + "##riz": 21885, + "##hul": 21886, + "corona": 21887, + "doubtful": 21888, + "niall": 21889, + "##pha": 21890, + "##bino": 21891, + "fines": 21892, + "cite": 21893, + "acknowledging": 21894, + "bangor": 21895, + "ballard": 21896, + "bathurst": 21897, + "##resh": 21898, + "huron": 21899, + "mustered": 21900, + "alzheimer": 21901, + "garments": 21902, + "kinase": 21903, + "tyre": 21904, + "warship": 21905, + "##cp": 21906, + "flashback": 21907, + "pulmonary": 21908, + "braun": 21909, + "cheat": 21910, + "kamal": 21911, + "cyclists": 21912, + "constructions": 21913, + "grenades": 21914, + "ndp": 21915, + "traveller": 21916, + "excuses": 21917, + "stomped": 21918, + "signalling": 21919, + "trimmed": 21920, + "futsal": 21921, + "mosques": 21922, + "relevance": 21923, + "##wine": 21924, + "wta": 21925, + "##23": 21926, + "##vah": 21927, + "##lter": 21928, + "hoc": 21929, + "##riding": 21930, + "optimistic": 21931, + "##´s": 21932, + "deco": 21933, + "sim": 21934, + "interacting": 21935, + "rejecting": 21936, + "moniker": 21937, + "waterways": 21938, + "##ieri": 21939, + "##oku": 21940, + "mayors": 21941, + "gdansk": 21942, + "outnumbered": 21943, + "pearls": 21944, + "##ended": 21945, + "##hampton": 21946, + "fairs": 21947, + "totals": 21948, + "dominating": 21949, + "262": 21950, + "notions": 21951, + "stairway": 21952, + "compiling": 21953, + "pursed": 21954, + "commodities": 21955, + "grease": 21956, + "yeast": 21957, + "##jong": 21958, + "carthage": 21959, + "griffiths": 21960, + "residual": 21961, + "amc": 21962, + "contraction": 21963, + "laird": 21964, + "sapphire": 21965, + "##marine": 21966, + "##ivated": 21967, + "amalgamation": 21968, + "dissolve": 21969, + "inclination": 21970, + "lyle": 21971, + "packaged": 21972, + "altitudes": 21973, + "suez": 21974, + "canons": 21975, + "graded": 21976, + "lurched": 21977, + "narrowing": 21978, + "boasts": 21979, + "guise": 21980, + "wed": 21981, + "enrico": 21982, + "##ovsky": 21983, + "rower": 21984, + "scarred": 21985, + "bree": 21986, + "cub": 21987, + "iberian": 21988, + "protagonists": 21989, + "bargaining": 21990, + "proposing": 21991, + "trainers": 21992, + "voyages": 21993, + "vans": 21994, + "fishes": 21995, + "##aea": 21996, + "##ivist": 21997, + "##verance": 21998, + "encryption": 21999, + "artworks": 22000, + "kazan": 22001, + "sabre": 22002, + "cleopatra": 22003, + "hepburn": 22004, + "rotting": 22005, + "supremacy": 22006, + "mecklenburg": 22007, + "##brate": 22008, + "burrows": 22009, + "hazards": 22010, + "outgoing": 22011, + "flair": 22012, + "organizes": 22013, + "##ctions": 22014, + "scorpion": 22015, + "##usions": 22016, + "boo": 22017, + "234": 22018, + "chevalier": 22019, + "dunedin": 22020, + "slapping": 22021, + "##34": 22022, + "ineligible": 22023, + "pensions": 22024, + "##38": 22025, + "##omic": 22026, + "manufactures": 22027, + "emails": 22028, + "bismarck": 22029, + "238": 22030, + "weakening": 22031, + "blackish": 22032, + "ding": 22033, + "mcgee": 22034, + "quo": 22035, + "##rling": 22036, + "northernmost": 22037, + "xx": 22038, + "manpower": 22039, + "greed": 22040, + "sampson": 22041, + "clicking": 22042, + "##ange": 22043, + "##horpe": 22044, + "##inations": 22045, + "##roving": 22046, + "torre": 22047, + "##eptive": 22048, + "##moral": 22049, + "symbolism": 22050, + "38th": 22051, + "asshole": 22052, + "meritorious": 22053, + "outfits": 22054, + "splashed": 22055, + "biographies": 22056, + "sprung": 22057, + "astros": 22058, + "##tale": 22059, + "302": 22060, + "737": 22061, + "filly": 22062, + "raoul": 22063, + "nw": 22064, + "tokugawa": 22065, + "linden": 22066, + "clubhouse": 22067, + "##apa": 22068, + "tracts": 22069, + "romano": 22070, + "##pio": 22071, + "putin": 22072, + "tags": 22073, + "##note": 22074, + "chained": 22075, + "dickson": 22076, + "gunshot": 22077, + "moe": 22078, + "gunn": 22079, + "rashid": 22080, + "##tails": 22081, + "zipper": 22082, + "##bas": 22083, + "##nea": 22084, + "contrasted": 22085, + "##ply": 22086, + "##udes": 22087, + "plum": 22088, + "pharaoh": 22089, + "##pile": 22090, + "aw": 22091, + "comedies": 22092, + "ingrid": 22093, + "sandwiches": 22094, + "subdivisions": 22095, + "1100": 22096, + "mariana": 22097, + "nokia": 22098, + "kamen": 22099, + "hz": 22100, + "delaney": 22101, + "veto": 22102, + "herring": 22103, + "##words": 22104, + "possessive": 22105, + "outlines": 22106, + "##roup": 22107, + "siemens": 22108, + "stairwell": 22109, + "rc": 22110, + "gallantry": 22111, + "messiah": 22112, + "palais": 22113, + "yells": 22114, + "233": 22115, + "zeppelin": 22116, + "##dm": 22117, + "bolivar": 22118, + "##cede": 22119, + "smackdown": 22120, + "mckinley": 22121, + "##mora": 22122, + "##yt": 22123, + "muted": 22124, + "geologic": 22125, + "finely": 22126, + "unitary": 22127, + "avatar": 22128, + "hamas": 22129, + "maynard": 22130, + "rees": 22131, + "bog": 22132, + "contrasting": 22133, + "##rut": 22134, + "liv": 22135, + "chico": 22136, + "disposition": 22137, + "pixel": 22138, + "##erate": 22139, + "becca": 22140, + "dmitry": 22141, + "yeshiva": 22142, + "narratives": 22143, + "##lva": 22144, + "##ulton": 22145, + "mercenary": 22146, + "sharpe": 22147, + "tempered": 22148, + "navigate": 22149, + "stealth": 22150, + "amassed": 22151, + "keynes": 22152, + "##lini": 22153, + "untouched": 22154, + "##rrie": 22155, + "havoc": 22156, + "lithium": 22157, + "##fighting": 22158, + "abyss": 22159, + "graf": 22160, + "southward": 22161, + "wolverine": 22162, + "balloons": 22163, + "implements": 22164, + "ngos": 22165, + "transitions": 22166, + "##icum": 22167, + "ambushed": 22168, + "concacaf": 22169, + "dormant": 22170, + "economists": 22171, + "##dim": 22172, + "costing": 22173, + "csi": 22174, + "rana": 22175, + "universite": 22176, + "boulders": 22177, + "verity": 22178, + "##llon": 22179, + "collin": 22180, + "mellon": 22181, + "misses": 22182, + "cypress": 22183, + "fluorescent": 22184, + "lifeless": 22185, + "spence": 22186, + "##ulla": 22187, + "crewe": 22188, + "shepard": 22189, + "pak": 22190, + "revelations": 22191, + "##م": 22192, + "jolly": 22193, + "gibbons": 22194, + "paw": 22195, + "##dro": 22196, + "##quel": 22197, + "freeing": 22198, + "##test": 22199, + "shack": 22200, + "fries": 22201, + "palatine": 22202, + "##51": 22203, + "##hiko": 22204, + "accompaniment": 22205, + "cruising": 22206, + "recycled": 22207, + "##aver": 22208, + "erwin": 22209, + "sorting": 22210, + "synthesizers": 22211, + "dyke": 22212, + "realities": 22213, + "sg": 22214, + "strides": 22215, + "enslaved": 22216, + "wetland": 22217, + "##ghan": 22218, + "competence": 22219, + "gunpowder": 22220, + "grassy": 22221, + "maroon": 22222, + "reactors": 22223, + "objection": 22224, + "##oms": 22225, + "carlson": 22226, + "gearbox": 22227, + "macintosh": 22228, + "radios": 22229, + "shelton": 22230, + "##sho": 22231, + "clergyman": 22232, + "prakash": 22233, + "254": 22234, + "mongols": 22235, + "trophies": 22236, + "oricon": 22237, + "228": 22238, + "stimuli": 22239, + "twenty20": 22240, + "cantonese": 22241, + "cortes": 22242, + "mirrored": 22243, + "##saurus": 22244, + "bhp": 22245, + "cristina": 22246, + "melancholy": 22247, + "##lating": 22248, + "enjoyable": 22249, + "nuevo": 22250, + "##wny": 22251, + "downfall": 22252, + "schumacher": 22253, + "##ind": 22254, + "banging": 22255, + "lausanne": 22256, + "rumbled": 22257, + "paramilitary": 22258, + "reflex": 22259, + "ax": 22260, + "amplitude": 22261, + "migratory": 22262, + "##gall": 22263, + "##ups": 22264, + "midi": 22265, + "barnard": 22266, + "lastly": 22267, + "sherry": 22268, + "##hp": 22269, + "##nall": 22270, + "keystone": 22271, + "##kra": 22272, + "carleton": 22273, + "slippery": 22274, + "##53": 22275, + "coloring": 22276, + "foe": 22277, + "socket": 22278, + "otter": 22279, + "##rgos": 22280, + "mats": 22281, + "##tose": 22282, + "consultants": 22283, + "bafta": 22284, + "bison": 22285, + "topping": 22286, + "##km": 22287, + "490": 22288, + "primal": 22289, + "abandonment": 22290, + "transplant": 22291, + "atoll": 22292, + "hideous": 22293, + "mort": 22294, + "pained": 22295, + "reproduced": 22296, + "tae": 22297, + "howling": 22298, + "##turn": 22299, + "unlawful": 22300, + "billionaire": 22301, + "hotter": 22302, + "poised": 22303, + "lansing": 22304, + "##chang": 22305, + "dinamo": 22306, + "retro": 22307, + "messing": 22308, + "nfc": 22309, + "domesday": 22310, + "##mina": 22311, + "blitz": 22312, + "timed": 22313, + "##athing": 22314, + "##kley": 22315, + "ascending": 22316, + "gesturing": 22317, + "##izations": 22318, + "signaled": 22319, + "tis": 22320, + "chinatown": 22321, + "mermaid": 22322, + "savanna": 22323, + "jameson": 22324, + "##aint": 22325, + "catalina": 22326, + "##pet": 22327, + "##hers": 22328, + "cochrane": 22329, + "cy": 22330, + "chatting": 22331, + "##kus": 22332, + "alerted": 22333, + "computation": 22334, + "mused": 22335, + "noelle": 22336, + "majestic": 22337, + "mohawk": 22338, + "campo": 22339, + "octagonal": 22340, + "##sant": 22341, + "##hend": 22342, + "241": 22343, + "aspiring": 22344, + "##mart": 22345, + "comprehend": 22346, + "iona": 22347, + "paralyzed": 22348, + "shimmering": 22349, + "swindon": 22350, + "rhone": 22351, + "##eley": 22352, + "reputed": 22353, + "configurations": 22354, + "pitchfork": 22355, + "agitation": 22356, + "francais": 22357, + "gillian": 22358, + "lipstick": 22359, + "##ilo": 22360, + "outsiders": 22361, + "pontifical": 22362, + "resisting": 22363, + "bitterness": 22364, + "sewer": 22365, + "rockies": 22366, + "##edd": 22367, + "##ucher": 22368, + "misleading": 22369, + "1756": 22370, + "exiting": 22371, + "galloway": 22372, + "##nging": 22373, + "risked": 22374, + "##heart": 22375, + "246": 22376, + "commemoration": 22377, + "schultz": 22378, + "##rka": 22379, + "integrating": 22380, + "##rsa": 22381, + "poses": 22382, + "shrieked": 22383, + "##weiler": 22384, + "guineas": 22385, + "gladys": 22386, + "jerking": 22387, + "owls": 22388, + "goldsmith": 22389, + "nightly": 22390, + "penetrating": 22391, + "##unced": 22392, + "lia": 22393, + "##33": 22394, + "ignited": 22395, + "betsy": 22396, + "##aring": 22397, + "##thorpe": 22398, + "follower": 22399, + "vigorously": 22400, + "##rave": 22401, + "coded": 22402, + "kiran": 22403, + "knit": 22404, + "zoology": 22405, + "tbilisi": 22406, + "##28": 22407, + "##bered": 22408, + "repository": 22409, + "govt": 22410, + "deciduous": 22411, + "dino": 22412, + "growling": 22413, + "##bba": 22414, + "enhancement": 22415, + "unleashed": 22416, + "chanting": 22417, + "pussy": 22418, + "biochemistry": 22419, + "##eric": 22420, + "kettle": 22421, + "repression": 22422, + "toxicity": 22423, + "nrhp": 22424, + "##arth": 22425, + "##kko": 22426, + "##bush": 22427, + "ernesto": 22428, + "commended": 22429, + "outspoken": 22430, + "242": 22431, + "mca": 22432, + "parchment": 22433, + "sms": 22434, + "kristen": 22435, + "##aton": 22436, + "bisexual": 22437, + "raked": 22438, + "glamour": 22439, + "navajo": 22440, + "a2": 22441, + "conditioned": 22442, + "showcased": 22443, + "##hma": 22444, + "spacious": 22445, + "youthful": 22446, + "##esa": 22447, + "usl": 22448, + "appliances": 22449, + "junta": 22450, + "brest": 22451, + "layne": 22452, + "conglomerate": 22453, + "enchanted": 22454, + "chao": 22455, + "loosened": 22456, + "picasso": 22457, + "circulating": 22458, + "inspect": 22459, + "montevideo": 22460, + "##centric": 22461, + "##kti": 22462, + "piazza": 22463, + "spurred": 22464, + "##aith": 22465, + "bari": 22466, + "freedoms": 22467, + "poultry": 22468, + "stamford": 22469, + "lieu": 22470, + "##ect": 22471, + "indigo": 22472, + "sarcastic": 22473, + "bahia": 22474, + "stump": 22475, + "attach": 22476, + "dvds": 22477, + "frankenstein": 22478, + "lille": 22479, + "approx": 22480, + "scriptures": 22481, + "pollen": 22482, + "##script": 22483, + "nmi": 22484, + "overseen": 22485, + "##ivism": 22486, + "tides": 22487, + "proponent": 22488, + "newmarket": 22489, + "inherit": 22490, + "milling": 22491, + "##erland": 22492, + "centralized": 22493, + "##rou": 22494, + "distributors": 22495, + "credentials": 22496, + "drawers": 22497, + "abbreviation": 22498, + "##lco": 22499, + "##xon": 22500, + "downing": 22501, + "uncomfortably": 22502, + "ripe": 22503, + "##oes": 22504, + "erase": 22505, + "franchises": 22506, + "##ever": 22507, + "populace": 22508, + "##bery": 22509, + "##khar": 22510, + "decomposition": 22511, + "pleas": 22512, + "##tet": 22513, + "daryl": 22514, + "sabah": 22515, + "##stle": 22516, + "##wide": 22517, + "fearless": 22518, + "genie": 22519, + "lesions": 22520, + "annette": 22521, + "##ogist": 22522, + "oboe": 22523, + "appendix": 22524, + "nair": 22525, + "dripped": 22526, + "petitioned": 22527, + "maclean": 22528, + "mosquito": 22529, + "parrot": 22530, + "rpg": 22531, + "hampered": 22532, + "1648": 22533, + "operatic": 22534, + "reservoirs": 22535, + "##tham": 22536, + "irrelevant": 22537, + "jolt": 22538, + "summarized": 22539, + "##fp": 22540, + "medallion": 22541, + "##taff": 22542, + "##−": 22543, + "clawed": 22544, + "harlow": 22545, + "narrower": 22546, + "goddard": 22547, + "marcia": 22548, + "bodied": 22549, + "fremont": 22550, + "suarez": 22551, + "altering": 22552, + "tempest": 22553, + "mussolini": 22554, + "porn": 22555, + "##isms": 22556, + "sweetly": 22557, + "oversees": 22558, + "walkers": 22559, + "solitude": 22560, + "grimly": 22561, + "shrines": 22562, + "hk": 22563, + "ich": 22564, + "supervisors": 22565, + "hostess": 22566, + "dietrich": 22567, + "legitimacy": 22568, + "brushes": 22569, + "expressive": 22570, + "##yp": 22571, + "dissipated": 22572, + "##rse": 22573, + "localized": 22574, + "systemic": 22575, + "##nikov": 22576, + "gettysburg": 22577, + "##js": 22578, + "##uaries": 22579, + "dialogues": 22580, + "muttering": 22581, + "251": 22582, + "housekeeper": 22583, + "sicilian": 22584, + "discouraged": 22585, + "##frey": 22586, + "beamed": 22587, + "kaladin": 22588, + "halftime": 22589, + "kidnap": 22590, + "##amo": 22591, + "##llet": 22592, + "1754": 22593, + "synonymous": 22594, + "depleted": 22595, + "instituto": 22596, + "insulin": 22597, + "reprised": 22598, + "##opsis": 22599, + "clashed": 22600, + "##ctric": 22601, + "interrupting": 22602, + "radcliffe": 22603, + "insisting": 22604, + "medici": 22605, + "1715": 22606, + "ejected": 22607, + "playfully": 22608, + "turbulent": 22609, + "##47": 22610, + "starvation": 22611, + "##rini": 22612, + "shipment": 22613, + "rebellious": 22614, + "petersen": 22615, + "verification": 22616, + "merits": 22617, + "##rified": 22618, + "cakes": 22619, + "##charged": 22620, + "1757": 22621, + "milford": 22622, + "shortages": 22623, + "spying": 22624, + "fidelity": 22625, + "##aker": 22626, + "emitted": 22627, + "storylines": 22628, + "harvested": 22629, + "seismic": 22630, + "##iform": 22631, + "cheung": 22632, + "kilda": 22633, + "theoretically": 22634, + "barbie": 22635, + "lynx": 22636, + "##rgy": 22637, + "##tius": 22638, + "goblin": 22639, + "mata": 22640, + "poisonous": 22641, + "##nburg": 22642, + "reactive": 22643, + "residues": 22644, + "obedience": 22645, + "##евич": 22646, + "conjecture": 22647, + "##rac": 22648, + "401": 22649, + "hating": 22650, + "sixties": 22651, + "kicker": 22652, + "moaning": 22653, + "motown": 22654, + "##bha": 22655, + "emancipation": 22656, + "neoclassical": 22657, + "##hering": 22658, + "consoles": 22659, + "ebert": 22660, + "professorship": 22661, + "##tures": 22662, + "sustaining": 22663, + "assaults": 22664, + "obeyed": 22665, + "affluent": 22666, + "incurred": 22667, + "tornadoes": 22668, + "##eber": 22669, + "##zow": 22670, + "emphasizing": 22671, + "highlanders": 22672, + "cheated": 22673, + "helmets": 22674, + "##ctus": 22675, + "internship": 22676, + "terence": 22677, + "bony": 22678, + "executions": 22679, + "legislators": 22680, + "berries": 22681, + "peninsular": 22682, + "tinged": 22683, + "##aco": 22684, + "1689": 22685, + "amplifier": 22686, + "corvette": 22687, + "ribbons": 22688, + "lavish": 22689, + "pennant": 22690, + "##lander": 22691, + "worthless": 22692, + "##chfield": 22693, + "##forms": 22694, + "mariano": 22695, + "pyrenees": 22696, + "expenditures": 22697, + "##icides": 22698, + "chesterfield": 22699, + "mandir": 22700, + "tailor": 22701, + "39th": 22702, + "sergey": 22703, + "nestled": 22704, + "willed": 22705, + "aristocracy": 22706, + "devotees": 22707, + "goodnight": 22708, + "raaf": 22709, + "rumored": 22710, + "weaponry": 22711, + "remy": 22712, + "appropriations": 22713, + "harcourt": 22714, + "burr": 22715, + "riaa": 22716, + "##lence": 22717, + "limitation": 22718, + "unnoticed": 22719, + "guo": 22720, + "soaking": 22721, + "swamps": 22722, + "##tica": 22723, + "collapsing": 22724, + "tatiana": 22725, + "descriptive": 22726, + "brigham": 22727, + "psalm": 22728, + "##chment": 22729, + "maddox": 22730, + "##lization": 22731, + "patti": 22732, + "caliph": 22733, + "##aja": 22734, + "akron": 22735, + "injuring": 22736, + "serra": 22737, + "##ganj": 22738, + "basins": 22739, + "##sari": 22740, + "astonished": 22741, + "launcher": 22742, + "##church": 22743, + "hilary": 22744, + "wilkins": 22745, + "sewing": 22746, + "##sf": 22747, + "stinging": 22748, + "##fia": 22749, + "##ncia": 22750, + "underwood": 22751, + "startup": 22752, + "##ition": 22753, + "compilations": 22754, + "vibrations": 22755, + "embankment": 22756, + "jurist": 22757, + "##nity": 22758, + "bard": 22759, + "juventus": 22760, + "groundwater": 22761, + "kern": 22762, + "palaces": 22763, + "helium": 22764, + "boca": 22765, + "cramped": 22766, + "marissa": 22767, + "soto": 22768, + "##worm": 22769, + "jae": 22770, + "princely": 22771, + "##ggy": 22772, + "faso": 22773, + "bazaar": 22774, + "warmly": 22775, + "##voking": 22776, + "229": 22777, + "pairing": 22778, + "##lite": 22779, + "##grate": 22780, + "##nets": 22781, + "wien": 22782, + "freaked": 22783, + "ulysses": 22784, + "rebirth": 22785, + "##alia": 22786, + "##rent": 22787, + "mummy": 22788, + "guzman": 22789, + "jimenez": 22790, + "stilled": 22791, + "##nitz": 22792, + "trajectory": 22793, + "tha": 22794, + "woken": 22795, + "archival": 22796, + "professions": 22797, + "##pts": 22798, + "##pta": 22799, + "hilly": 22800, + "shadowy": 22801, + "shrink": 22802, + "##bolt": 22803, + "norwood": 22804, + "glued": 22805, + "migrate": 22806, + "stereotypes": 22807, + "devoid": 22808, + "##pheus": 22809, + "625": 22810, + "evacuate": 22811, + "horrors": 22812, + "infancy": 22813, + "gotham": 22814, + "knowles": 22815, + "optic": 22816, + "downloaded": 22817, + "sachs": 22818, + "kingsley": 22819, + "parramatta": 22820, + "darryl": 22821, + "mor": 22822, + "##onale": 22823, + "shady": 22824, + "commence": 22825, + "confesses": 22826, + "kan": 22827, + "##meter": 22828, + "##placed": 22829, + "marlborough": 22830, + "roundabout": 22831, + "regents": 22832, + "frigates": 22833, + "io": 22834, + "##imating": 22835, + "gothenburg": 22836, + "revoked": 22837, + "carvings": 22838, + "clockwise": 22839, + "convertible": 22840, + "intruder": 22841, + "##sche": 22842, + "banged": 22843, + "##ogo": 22844, + "vicky": 22845, + "bourgeois": 22846, + "##mony": 22847, + "dupont": 22848, + "footing": 22849, + "##gum": 22850, + "pd": 22851, + "##real": 22852, + "buckle": 22853, + "yun": 22854, + "penthouse": 22855, + "sane": 22856, + "720": 22857, + "serviced": 22858, + "stakeholders": 22859, + "neumann": 22860, + "bb": 22861, + "##eers": 22862, + "comb": 22863, + "##gam": 22864, + "catchment": 22865, + "pinning": 22866, + "rallies": 22867, + "typing": 22868, + "##elles": 22869, + "forefront": 22870, + "freiburg": 22871, + "sweetie": 22872, + "giacomo": 22873, + "widowed": 22874, + "goodwill": 22875, + "worshipped": 22876, + "aspirations": 22877, + "midday": 22878, + "##vat": 22879, + "fishery": 22880, + "##trick": 22881, + "bournemouth": 22882, + "turk": 22883, + "243": 22884, + "hearth": 22885, + "ethanol": 22886, + "guadalajara": 22887, + "murmurs": 22888, + "sl": 22889, + "##uge": 22890, + "afforded": 22891, + "scripted": 22892, + "##hta": 22893, + "wah": 22894, + "##jn": 22895, + "coroner": 22896, + "translucent": 22897, + "252": 22898, + "memorials": 22899, + "puck": 22900, + "progresses": 22901, + "clumsy": 22902, + "##race": 22903, + "315": 22904, + "candace": 22905, + "recounted": 22906, + "##27": 22907, + "##slin": 22908, + "##uve": 22909, + "filtering": 22910, + "##mac": 22911, + "howl": 22912, + "strata": 22913, + "heron": 22914, + "leveled": 22915, + "##ays": 22916, + "dubious": 22917, + "##oja": 22918, + "##т": 22919, + "##wheel": 22920, + "citations": 22921, + "exhibiting": 22922, + "##laya": 22923, + "##mics": 22924, + "##pods": 22925, + "turkic": 22926, + "##lberg": 22927, + "injunction": 22928, + "##ennial": 22929, + "##mit": 22930, + "antibodies": 22931, + "##44": 22932, + "organise": 22933, + "##rigues": 22934, + "cardiovascular": 22935, + "cushion": 22936, + "inverness": 22937, + "##zquez": 22938, + "dia": 22939, + "cocoa": 22940, + "sibling": 22941, + "##tman": 22942, + "##roid": 22943, + "expanse": 22944, + "feasible": 22945, + "tunisian": 22946, + "algiers": 22947, + "##relli": 22948, + "rus": 22949, + "bloomberg": 22950, + "dso": 22951, + "westphalia": 22952, + "bro": 22953, + "tacoma": 22954, + "281": 22955, + "downloads": 22956, + "##ours": 22957, + "konrad": 22958, + "duran": 22959, + "##hdi": 22960, + "continuum": 22961, + "jett": 22962, + "compares": 22963, + "legislator": 22964, + "secession": 22965, + "##nable": 22966, + "##gues": 22967, + "##zuka": 22968, + "translating": 22969, + "reacher": 22970, + "##gley": 22971, + "##ła": 22972, + "aleppo": 22973, + "##agi": 22974, + "tc": 22975, + "orchards": 22976, + "trapping": 22977, + "linguist": 22978, + "versatile": 22979, + "drumming": 22980, + "postage": 22981, + "calhoun": 22982, + "superiors": 22983, + "##mx": 22984, + "barefoot": 22985, + "leary": 22986, + "##cis": 22987, + "ignacio": 22988, + "alfa": 22989, + "kaplan": 22990, + "##rogen": 22991, + "bratislava": 22992, + "mori": 22993, + "##vot": 22994, + "disturb": 22995, + "haas": 22996, + "313": 22997, + "cartridges": 22998, + "gilmore": 22999, + "radiated": 23000, + "salford": 23001, + "tunic": 23002, + "hades": 23003, + "##ulsive": 23004, + "archeological": 23005, + "delilah": 23006, + "magistrates": 23007, + "auditioned": 23008, + "brewster": 23009, + "charters": 23010, + "empowerment": 23011, + "blogs": 23012, + "cappella": 23013, + "dynasties": 23014, + "iroquois": 23015, + "whipping": 23016, + "##krishna": 23017, + "raceway": 23018, + "truths": 23019, + "myra": 23020, + "weaken": 23021, + "judah": 23022, + "mcgregor": 23023, + "##horse": 23024, + "mic": 23025, + "refueling": 23026, + "37th": 23027, + "burnley": 23028, + "bosses": 23029, + "markus": 23030, + "premio": 23031, + "query": 23032, + "##gga": 23033, + "dunbar": 23034, + "##economic": 23035, + "darkest": 23036, + "lyndon": 23037, + "sealing": 23038, + "commendation": 23039, + "reappeared": 23040, + "##mun": 23041, + "addicted": 23042, + "ezio": 23043, + "slaughtered": 23044, + "satisfactory": 23045, + "shuffle": 23046, + "##eves": 23047, + "##thic": 23048, + "##uj": 23049, + "fortification": 23050, + "warrington": 23051, + "##otto": 23052, + "resurrected": 23053, + "fargo": 23054, + "mane": 23055, + "##utable": 23056, + "##lei": 23057, + "##space": 23058, + "foreword": 23059, + "ox": 23060, + "##aris": 23061, + "##vern": 23062, + "abrams": 23063, + "hua": 23064, + "##mento": 23065, + "sakura": 23066, + "##alo": 23067, + "uv": 23068, + "sentimental": 23069, + "##skaya": 23070, + "midfield": 23071, + "##eses": 23072, + "sturdy": 23073, + "scrolls": 23074, + "macleod": 23075, + "##kyu": 23076, + "entropy": 23077, + "##lance": 23078, + "mitochondrial": 23079, + "cicero": 23080, + "excelled": 23081, + "thinner": 23082, + "convoys": 23083, + "perceive": 23084, + "##oslav": 23085, + "##urable": 23086, + "systematically": 23087, + "grind": 23088, + "burkina": 23089, + "287": 23090, + "##tagram": 23091, + "ops": 23092, + "##aman": 23093, + "guantanamo": 23094, + "##cloth": 23095, + "##tite": 23096, + "forcefully": 23097, + "wavy": 23098, + "##jou": 23099, + "pointless": 23100, + "##linger": 23101, + "##tze": 23102, + "layton": 23103, + "portico": 23104, + "superficial": 23105, + "clerical": 23106, + "outlaws": 23107, + "##hism": 23108, + "burials": 23109, + "muir": 23110, + "##inn": 23111, + "creditors": 23112, + "hauling": 23113, + "rattle": 23114, + "##leg": 23115, + "calais": 23116, + "monde": 23117, + "archers": 23118, + "reclaimed": 23119, + "dwell": 23120, + "wexford": 23121, + "hellenic": 23122, + "falsely": 23123, + "remorse": 23124, + "##tek": 23125, + "dough": 23126, + "furnishings": 23127, + "##uttered": 23128, + "gabon": 23129, + "neurological": 23130, + "novice": 23131, + "##igraphy": 23132, + "contemplated": 23133, + "pulpit": 23134, + "nightstand": 23135, + "saratoga": 23136, + "##istan": 23137, + "documenting": 23138, + "pulsing": 23139, + "taluk": 23140, + "##firmed": 23141, + "busted": 23142, + "marital": 23143, + "##rien": 23144, + "disagreements": 23145, + "wasps": 23146, + "##yes": 23147, + "hodge": 23148, + "mcdonnell": 23149, + "mimic": 23150, + "fran": 23151, + "pendant": 23152, + "dhabi": 23153, + "musa": 23154, + "##nington": 23155, + "congratulations": 23156, + "argent": 23157, + "darrell": 23158, + "concussion": 23159, + "losers": 23160, + "regrets": 23161, + "thessaloniki": 23162, + "reversal": 23163, + "donaldson": 23164, + "hardwood": 23165, + "thence": 23166, + "achilles": 23167, + "ritter": 23168, + "##eran": 23169, + "demonic": 23170, + "jurgen": 23171, + "prophets": 23172, + "goethe": 23173, + "eki": 23174, + "classmate": 23175, + "buff": 23176, + "##cking": 23177, + "yank": 23178, + "irrational": 23179, + "##inging": 23180, + "perished": 23181, + "seductive": 23182, + "qur": 23183, + "sourced": 23184, + "##crat": 23185, + "##typic": 23186, + "mustard": 23187, + "ravine": 23188, + "barre": 23189, + "horizontally": 23190, + "characterization": 23191, + "phylogenetic": 23192, + "boise": 23193, + "##dit": 23194, + "##runner": 23195, + "##tower": 23196, + "brutally": 23197, + "intercourse": 23198, + "seduce": 23199, + "##bbing": 23200, + "fay": 23201, + "ferris": 23202, + "ogden": 23203, + "amar": 23204, + "nik": 23205, + "unarmed": 23206, + "##inator": 23207, + "evaluating": 23208, + "kyrgyzstan": 23209, + "sweetness": 23210, + "##lford": 23211, + "##oki": 23212, + "mccormick": 23213, + "meiji": 23214, + "notoriety": 23215, + "stimulate": 23216, + "disrupt": 23217, + "figuring": 23218, + "instructional": 23219, + "mcgrath": 23220, + "##zoo": 23221, + "groundbreaking": 23222, + "##lto": 23223, + "flinch": 23224, + "khorasan": 23225, + "agrarian": 23226, + "bengals": 23227, + "mixer": 23228, + "radiating": 23229, + "##sov": 23230, + "ingram": 23231, + "pitchers": 23232, + "nad": 23233, + "tariff": 23234, + "##cript": 23235, + "tata": 23236, + "##codes": 23237, + "##emi": 23238, + "##ungen": 23239, + "appellate": 23240, + "lehigh": 23241, + "##bled": 23242, + "##giri": 23243, + "brawl": 23244, + "duct": 23245, + "texans": 23246, + "##ciation": 23247, + "##ropolis": 23248, + "skipper": 23249, + "speculative": 23250, + "vomit": 23251, + "doctrines": 23252, + "stresses": 23253, + "253": 23254, + "davy": 23255, + "graders": 23256, + "whitehead": 23257, + "jozef": 23258, + "timely": 23259, + "cumulative": 23260, + "haryana": 23261, + "paints": 23262, + "appropriately": 23263, + "boon": 23264, + "cactus": 23265, + "##ales": 23266, + "##pid": 23267, + "dow": 23268, + "legions": 23269, + "##pit": 23270, + "perceptions": 23271, + "1730": 23272, + "picturesque": 23273, + "##yse": 23274, + "periphery": 23275, + "rune": 23276, + "wr": 23277, + "##aha": 23278, + "celtics": 23279, + "sentencing": 23280, + "whoa": 23281, + "##erin": 23282, + "confirms": 23283, + "variance": 23284, + "425": 23285, + "moines": 23286, + "mathews": 23287, + "spade": 23288, + "rave": 23289, + "m1": 23290, + "fronted": 23291, + "fx": 23292, + "blending": 23293, + "alleging": 23294, + "reared": 23295, + "##gl": 23296, + "237": 23297, + "##paper": 23298, + "grassroots": 23299, + "eroded": 23300, + "##free": 23301, + "##physical": 23302, + "directs": 23303, + "ordeal": 23304, + "##sław": 23305, + "accelerate": 23306, + "hacker": 23307, + "rooftop": 23308, + "##inia": 23309, + "lev": 23310, + "buys": 23311, + "cebu": 23312, + "devote": 23313, + "##lce": 23314, + "specialising": 23315, + "##ulsion": 23316, + "choreographed": 23317, + "repetition": 23318, + "warehouses": 23319, + "##ryl": 23320, + "paisley": 23321, + "tuscany": 23322, + "analogy": 23323, + "sorcerer": 23324, + "hash": 23325, + "huts": 23326, + "shards": 23327, + "descends": 23328, + "exclude": 23329, + "nix": 23330, + "chaplin": 23331, + "gaga": 23332, + "ito": 23333, + "vane": 23334, + "##drich": 23335, + "causeway": 23336, + "misconduct": 23337, + "limo": 23338, + "orchestrated": 23339, + "glands": 23340, + "jana": 23341, + "##kot": 23342, + "u2": 23343, + "##mple": 23344, + "##sons": 23345, + "branching": 23346, + "contrasts": 23347, + "scoop": 23348, + "longed": 23349, + "##virus": 23350, + "chattanooga": 23351, + "##75": 23352, + "syrup": 23353, + "cornerstone": 23354, + "##tized": 23355, + "##mind": 23356, + "##iaceae": 23357, + "careless": 23358, + "precedence": 23359, + "frescoes": 23360, + "##uet": 23361, + "chilled": 23362, + "consult": 23363, + "modelled": 23364, + "snatch": 23365, + "peat": 23366, + "##thermal": 23367, + "caucasian": 23368, + "humane": 23369, + "relaxation": 23370, + "spins": 23371, + "temperance": 23372, + "##lbert": 23373, + "occupations": 23374, + "lambda": 23375, + "hybrids": 23376, + "moons": 23377, + "mp3": 23378, + "##oese": 23379, + "247": 23380, + "rolf": 23381, + "societal": 23382, + "yerevan": 23383, + "ness": 23384, + "##ssler": 23385, + "befriended": 23386, + "mechanized": 23387, + "nominate": 23388, + "trough": 23389, + "boasted": 23390, + "cues": 23391, + "seater": 23392, + "##hom": 23393, + "bends": 23394, + "##tangle": 23395, + "conductors": 23396, + "emptiness": 23397, + "##lmer": 23398, + "eurasian": 23399, + "adriatic": 23400, + "tian": 23401, + "##cie": 23402, + "anxiously": 23403, + "lark": 23404, + "propellers": 23405, + "chichester": 23406, + "jock": 23407, + "ev": 23408, + "2a": 23409, + "##holding": 23410, + "credible": 23411, + "recounts": 23412, + "tori": 23413, + "loyalist": 23414, + "abduction": 23415, + "##hoot": 23416, + "##redo": 23417, + "nepali": 23418, + "##mite": 23419, + "ventral": 23420, + "tempting": 23421, + "##ango": 23422, + "##crats": 23423, + "steered": 23424, + "##wice": 23425, + "javelin": 23426, + "dipping": 23427, + "laborers": 23428, + "prentice": 23429, + "looming": 23430, + "titanium": 23431, + "##ː": 23432, + "badges": 23433, + "emir": 23434, + "tensor": 23435, + "##ntation": 23436, + "egyptians": 23437, + "rash": 23438, + "denies": 23439, + "hawthorne": 23440, + "lombard": 23441, + "showers": 23442, + "wehrmacht": 23443, + "dietary": 23444, + "trojan": 23445, + "##reus": 23446, + "welles": 23447, + "executing": 23448, + "horseshoe": 23449, + "lifeboat": 23450, + "##lak": 23451, + "elsa": 23452, + "infirmary": 23453, + "nearing": 23454, + "roberta": 23455, + "boyer": 23456, + "mutter": 23457, + "trillion": 23458, + "joanne": 23459, + "##fine": 23460, + "##oked": 23461, + "sinks": 23462, + "vortex": 23463, + "uruguayan": 23464, + "clasp": 23465, + "sirius": 23466, + "##block": 23467, + "accelerator": 23468, + "prohibit": 23469, + "sunken": 23470, + "byu": 23471, + "chronological": 23472, + "diplomats": 23473, + "ochreous": 23474, + "510": 23475, + "symmetrical": 23476, + "1644": 23477, + "maia": 23478, + "##tology": 23479, + "salts": 23480, + "reigns": 23481, + "atrocities": 23482, + "##ия": 23483, + "hess": 23484, + "bared": 23485, + "issn": 23486, + "##vyn": 23487, + "cater": 23488, + "saturated": 23489, + "##cycle": 23490, + "##isse": 23491, + "sable": 23492, + "voyager": 23493, + "dyer": 23494, + "yusuf": 23495, + "##inge": 23496, + "fountains": 23497, + "wolff": 23498, + "##39": 23499, + "##nni": 23500, + "engraving": 23501, + "rollins": 23502, + "atheist": 23503, + "ominous": 23504, + "##ault": 23505, + "herr": 23506, + "chariot": 23507, + "martina": 23508, + "strung": 23509, + "##fell": 23510, + "##farlane": 23511, + "horrific": 23512, + "sahib": 23513, + "gazes": 23514, + "saetan": 23515, + "erased": 23516, + "ptolemy": 23517, + "##olic": 23518, + "flushing": 23519, + "lauderdale": 23520, + "analytic": 23521, + "##ices": 23522, + "530": 23523, + "navarro": 23524, + "beak": 23525, + "gorilla": 23526, + "herrera": 23527, + "broom": 23528, + "guadalupe": 23529, + "raiding": 23530, + "sykes": 23531, + "311": 23532, + "bsc": 23533, + "deliveries": 23534, + "1720": 23535, + "invasions": 23536, + "carmichael": 23537, + "tajikistan": 23538, + "thematic": 23539, + "ecumenical": 23540, + "sentiments": 23541, + "onstage": 23542, + "##rians": 23543, + "##brand": 23544, + "##sume": 23545, + "catastrophic": 23546, + "flanks": 23547, + "molten": 23548, + "##arns": 23549, + "waller": 23550, + "aimee": 23551, + "terminating": 23552, + "##icing": 23553, + "alternately": 23554, + "##oche": 23555, + "nehru": 23556, + "printers": 23557, + "outraged": 23558, + "##eving": 23559, + "empires": 23560, + "template": 23561, + "banners": 23562, + "repetitive": 23563, + "za": 23564, + "##oise": 23565, + "vegetarian": 23566, + "##tell": 23567, + "guiana": 23568, + "opt": 23569, + "cavendish": 23570, + "lucknow": 23571, + "synthesized": 23572, + "##hani": 23573, + "##mada": 23574, + "finalized": 23575, + "##ctable": 23576, + "fictitious": 23577, + "mayoral": 23578, + "unreliable": 23579, + "##enham": 23580, + "embracing": 23581, + "peppers": 23582, + "rbis": 23583, + "##chio": 23584, + "##neo": 23585, + "inhibition": 23586, + "slashed": 23587, + "togo": 23588, + "orderly": 23589, + "embroidered": 23590, + "safari": 23591, + "salty": 23592, + "236": 23593, + "barron": 23594, + "benito": 23595, + "totaled": 23596, + "##dak": 23597, + "pubs": 23598, + "simulated": 23599, + "caden": 23600, + "devin": 23601, + "tolkien": 23602, + "momma": 23603, + "welding": 23604, + "sesame": 23605, + "##ept": 23606, + "gottingen": 23607, + "hardness": 23608, + "630": 23609, + "shaman": 23610, + "temeraire": 23611, + "620": 23612, + "adequately": 23613, + "pediatric": 23614, + "##kit": 23615, + "ck": 23616, + "assertion": 23617, + "radicals": 23618, + "composure": 23619, + "cadence": 23620, + "seafood": 23621, + "beaufort": 23622, + "lazarus": 23623, + "mani": 23624, + "warily": 23625, + "cunning": 23626, + "kurdistan": 23627, + "249": 23628, + "cantata": 23629, + "##kir": 23630, + "ares": 23631, + "##41": 23632, + "##clusive": 23633, + "nape": 23634, + "townland": 23635, + "geared": 23636, + "insulted": 23637, + "flutter": 23638, + "boating": 23639, + "violate": 23640, + "draper": 23641, + "dumping": 23642, + "malmo": 23643, + "##hh": 23644, + "##romatic": 23645, + "firearm": 23646, + "alta": 23647, + "bono": 23648, + "obscured": 23649, + "##clave": 23650, + "exceeds": 23651, + "panorama": 23652, + "unbelievable": 23653, + "##train": 23654, + "preschool": 23655, + "##essed": 23656, + "disconnected": 23657, + "installing": 23658, + "rescuing": 23659, + "secretaries": 23660, + "accessibility": 23661, + "##castle": 23662, + "##drive": 23663, + "##ifice": 23664, + "##film": 23665, + "bouts": 23666, + "slug": 23667, + "waterway": 23668, + "mindanao": 23669, + "##buro": 23670, + "##ratic": 23671, + "halves": 23672, + "##ل": 23673, + "calming": 23674, + "liter": 23675, + "maternity": 23676, + "adorable": 23677, + "bragg": 23678, + "electrification": 23679, + "mcc": 23680, + "##dote": 23681, + "roxy": 23682, + "schizophrenia": 23683, + "##body": 23684, + "munoz": 23685, + "kaye": 23686, + "whaling": 23687, + "239": 23688, + "mil": 23689, + "tingling": 23690, + "tolerant": 23691, + "##ago": 23692, + "unconventional": 23693, + "volcanoes": 23694, + "##finder": 23695, + "deportivo": 23696, + "##llie": 23697, + "robson": 23698, + "kaufman": 23699, + "neuroscience": 23700, + "wai": 23701, + "deportation": 23702, + "masovian": 23703, + "scraping": 23704, + "converse": 23705, + "##bh": 23706, + "hacking": 23707, + "bulge": 23708, + "##oun": 23709, + "administratively": 23710, + "yao": 23711, + "580": 23712, + "amp": 23713, + "mammoth": 23714, + "booster": 23715, + "claremont": 23716, + "hooper": 23717, + "nomenclature": 23718, + "pursuits": 23719, + "mclaughlin": 23720, + "melinda": 23721, + "##sul": 23722, + "catfish": 23723, + "barclay": 23724, + "substrates": 23725, + "taxa": 23726, + "zee": 23727, + "originals": 23728, + "kimberly": 23729, + "packets": 23730, + "padma": 23731, + "##ality": 23732, + "borrowing": 23733, + "ostensibly": 23734, + "solvent": 23735, + "##bri": 23736, + "##genesis": 23737, + "##mist": 23738, + "lukas": 23739, + "shreveport": 23740, + "veracruz": 23741, + "##ь": 23742, + "##lou": 23743, + "##wives": 23744, + "cheney": 23745, + "tt": 23746, + "anatolia": 23747, + "hobbs": 23748, + "##zyn": 23749, + "cyclic": 23750, + "radiant": 23751, + "alistair": 23752, + "greenish": 23753, + "siena": 23754, + "dat": 23755, + "independents": 23756, + "##bation": 23757, + "conform": 23758, + "pieter": 23759, + "hyper": 23760, + "applicant": 23761, + "bradshaw": 23762, + "spores": 23763, + "telangana": 23764, + "vinci": 23765, + "inexpensive": 23766, + "nuclei": 23767, + "322": 23768, + "jang": 23769, + "nme": 23770, + "soho": 23771, + "spd": 23772, + "##ign": 23773, + "cradled": 23774, + "receptionist": 23775, + "pow": 23776, + "##43": 23777, + "##rika": 23778, + "fascism": 23779, + "##ifer": 23780, + "experimenting": 23781, + "##ading": 23782, + "##iec": 23783, + "##region": 23784, + "345": 23785, + "jocelyn": 23786, + "maris": 23787, + "stair": 23788, + "nocturnal": 23789, + "toro": 23790, + "constabulary": 23791, + "elgin": 23792, + "##kker": 23793, + "msc": 23794, + "##giving": 23795, + "##schen": 23796, + "##rase": 23797, + "doherty": 23798, + "doping": 23799, + "sarcastically": 23800, + "batter": 23801, + "maneuvers": 23802, + "##cano": 23803, + "##apple": 23804, + "##gai": 23805, + "##git": 23806, + "intrinsic": 23807, + "##nst": 23808, + "##stor": 23809, + "1753": 23810, + "showtime": 23811, + "cafes": 23812, + "gasps": 23813, + "lviv": 23814, + "ushered": 23815, + "##thed": 23816, + "fours": 23817, + "restart": 23818, + "astonishment": 23819, + "transmitting": 23820, + "flyer": 23821, + "shrugs": 23822, + "##sau": 23823, + "intriguing": 23824, + "cones": 23825, + "dictated": 23826, + "mushrooms": 23827, + "medial": 23828, + "##kovsky": 23829, + "##elman": 23830, + "escorting": 23831, + "gaped": 23832, + "##26": 23833, + "godfather": 23834, + "##door": 23835, + "##sell": 23836, + "djs": 23837, + "recaptured": 23838, + "timetable": 23839, + "vila": 23840, + "1710": 23841, + "3a": 23842, + "aerodrome": 23843, + "mortals": 23844, + "scientology": 23845, + "##orne": 23846, + "angelina": 23847, + "mag": 23848, + "convection": 23849, + "unpaid": 23850, + "insertion": 23851, + "intermittent": 23852, + "lego": 23853, + "##nated": 23854, + "endeavor": 23855, + "kota": 23856, + "pereira": 23857, + "##lz": 23858, + "304": 23859, + "bwv": 23860, + "glamorgan": 23861, + "insults": 23862, + "agatha": 23863, + "fey": 23864, + "##cend": 23865, + "fleetwood": 23866, + "mahogany": 23867, + "protruding": 23868, + "steamship": 23869, + "zeta": 23870, + "##arty": 23871, + "mcguire": 23872, + "suspense": 23873, + "##sphere": 23874, + "advising": 23875, + "urges": 23876, + "##wala": 23877, + "hurriedly": 23878, + "meteor": 23879, + "gilded": 23880, + "inline": 23881, + "arroyo": 23882, + "stalker": 23883, + "##oge": 23884, + "excitedly": 23885, + "revered": 23886, + "##cure": 23887, + "earle": 23888, + "introductory": 23889, + "##break": 23890, + "##ilde": 23891, + "mutants": 23892, + "puff": 23893, + "pulses": 23894, + "reinforcement": 23895, + "##haling": 23896, + "curses": 23897, + "lizards": 23898, + "stalk": 23899, + "correlated": 23900, + "##fixed": 23901, + "fallout": 23902, + "macquarie": 23903, + "##unas": 23904, + "bearded": 23905, + "denton": 23906, + "heaving": 23907, + "802": 23908, + "##ocation": 23909, + "winery": 23910, + "assign": 23911, + "dortmund": 23912, + "##lkirk": 23913, + "everest": 23914, + "invariant": 23915, + "charismatic": 23916, + "susie": 23917, + "##elling": 23918, + "bled": 23919, + "lesley": 23920, + "telegram": 23921, + "sumner": 23922, + "bk": 23923, + "##ogen": 23924, + "##к": 23925, + "wilcox": 23926, + "needy": 23927, + "colbert": 23928, + "duval": 23929, + "##iferous": 23930, + "##mbled": 23931, + "allotted": 23932, + "attends": 23933, + "imperative": 23934, + "##hita": 23935, + "replacements": 23936, + "hawker": 23937, + "##inda": 23938, + "insurgency": 23939, + "##zee": 23940, + "##eke": 23941, + "casts": 23942, + "##yla": 23943, + "680": 23944, + "ives": 23945, + "transitioned": 23946, + "##pack": 23947, + "##powering": 23948, + "authoritative": 23949, + "baylor": 23950, + "flex": 23951, + "cringed": 23952, + "plaintiffs": 23953, + "woodrow": 23954, + "##skie": 23955, + "drastic": 23956, + "ape": 23957, + "aroma": 23958, + "unfolded": 23959, + "commotion": 23960, + "nt": 23961, + "preoccupied": 23962, + "theta": 23963, + "routines": 23964, + "lasers": 23965, + "privatization": 23966, + "wand": 23967, + "domino": 23968, + "ek": 23969, + "clenching": 23970, + "nsa": 23971, + "strategically": 23972, + "showered": 23973, + "bile": 23974, + "handkerchief": 23975, + "pere": 23976, + "storing": 23977, + "christophe": 23978, + "insulting": 23979, + "316": 23980, + "nakamura": 23981, + "romani": 23982, + "asiatic": 23983, + "magdalena": 23984, + "palma": 23985, + "cruises": 23986, + "stripping": 23987, + "405": 23988, + "konstantin": 23989, + "soaring": 23990, + "##berman": 23991, + "colloquially": 23992, + "forerunner": 23993, + "havilland": 23994, + "incarcerated": 23995, + "parasites": 23996, + "sincerity": 23997, + "##utus": 23998, + "disks": 23999, + "plank": 24000, + "saigon": 24001, + "##ining": 24002, + "corbin": 24003, + "homo": 24004, + "ornaments": 24005, + "powerhouse": 24006, + "##tlement": 24007, + "chong": 24008, + "fastened": 24009, + "feasibility": 24010, + "idf": 24011, + "morphological": 24012, + "usable": 24013, + "##nish": 24014, + "##zuki": 24015, + "aqueduct": 24016, + "jaguars": 24017, + "keepers": 24018, + "##flies": 24019, + "aleksandr": 24020, + "faust": 24021, + "assigns": 24022, + "ewing": 24023, + "bacterium": 24024, + "hurled": 24025, + "tricky": 24026, + "hungarians": 24027, + "integers": 24028, + "wallis": 24029, + "321": 24030, + "yamaha": 24031, + "##isha": 24032, + "hushed": 24033, + "oblivion": 24034, + "aviator": 24035, + "evangelist": 24036, + "friars": 24037, + "##eller": 24038, + "monograph": 24039, + "ode": 24040, + "##nary": 24041, + "airplanes": 24042, + "labourers": 24043, + "charms": 24044, + "##nee": 24045, + "1661": 24046, + "hagen": 24047, + "tnt": 24048, + "rudder": 24049, + "fiesta": 24050, + "transcript": 24051, + "dorothea": 24052, + "ska": 24053, + "inhibitor": 24054, + "maccabi": 24055, + "retorted": 24056, + "raining": 24057, + "encompassed": 24058, + "clauses": 24059, + "menacing": 24060, + "1642": 24061, + "lineman": 24062, + "##gist": 24063, + "vamps": 24064, + "##ape": 24065, + "##dick": 24066, + "gloom": 24067, + "##rera": 24068, + "dealings": 24069, + "easing": 24070, + "seekers": 24071, + "##nut": 24072, + "##pment": 24073, + "helens": 24074, + "unmanned": 24075, + "##anu": 24076, + "##isson": 24077, + "basics": 24078, + "##amy": 24079, + "##ckman": 24080, + "adjustments": 24081, + "1688": 24082, + "brutality": 24083, + "horne": 24084, + "##zell": 24085, + "sui": 24086, + "##55": 24087, + "##mable": 24088, + "aggregator": 24089, + "##thal": 24090, + "rhino": 24091, + "##drick": 24092, + "##vira": 24093, + "counters": 24094, + "zoom": 24095, + "##01": 24096, + "##rting": 24097, + "mn": 24098, + "montenegrin": 24099, + "packard": 24100, + "##unciation": 24101, + "##♭": 24102, + "##kki": 24103, + "reclaim": 24104, + "scholastic": 24105, + "thugs": 24106, + "pulsed": 24107, + "##icia": 24108, + "syriac": 24109, + "quan": 24110, + "saddam": 24111, + "banda": 24112, + "kobe": 24113, + "blaming": 24114, + "buddies": 24115, + "dissent": 24116, + "##lusion": 24117, + "##usia": 24118, + "corbett": 24119, + "jaya": 24120, + "delle": 24121, + "erratic": 24122, + "lexie": 24123, + "##hesis": 24124, + "435": 24125, + "amiga": 24126, + "hermes": 24127, + "##pressing": 24128, + "##leen": 24129, + "chapels": 24130, + "gospels": 24131, + "jamal": 24132, + "##uating": 24133, + "compute": 24134, + "revolving": 24135, + "warp": 24136, + "##sso": 24137, + "##thes": 24138, + "armory": 24139, + "##eras": 24140, + "##gol": 24141, + "antrim": 24142, + "loki": 24143, + "##kow": 24144, + "##asian": 24145, + "##good": 24146, + "##zano": 24147, + "braid": 24148, + "handwriting": 24149, + "subdistrict": 24150, + "funky": 24151, + "pantheon": 24152, + "##iculate": 24153, + "concurrency": 24154, + "estimation": 24155, + "improper": 24156, + "juliana": 24157, + "##his": 24158, + "newcomers": 24159, + "johnstone": 24160, + "staten": 24161, + "communicated": 24162, + "##oco": 24163, + "##alle": 24164, + "sausage": 24165, + "stormy": 24166, + "##stered": 24167, + "##tters": 24168, + "superfamily": 24169, + "##grade": 24170, + "acidic": 24171, + "collateral": 24172, + "tabloid": 24173, + "##oped": 24174, + "##rza": 24175, + "bladder": 24176, + "austen": 24177, + "##ellant": 24178, + "mcgraw": 24179, + "##hay": 24180, + "hannibal": 24181, + "mein": 24182, + "aquino": 24183, + "lucifer": 24184, + "wo": 24185, + "badger": 24186, + "boar": 24187, + "cher": 24188, + "christensen": 24189, + "greenberg": 24190, + "interruption": 24191, + "##kken": 24192, + "jem": 24193, + "244": 24194, + "mocked": 24195, + "bottoms": 24196, + "cambridgeshire": 24197, + "##lide": 24198, + "sprawling": 24199, + "##bbly": 24200, + "eastwood": 24201, + "ghent": 24202, + "synth": 24203, + "##buck": 24204, + "advisers": 24205, + "##bah": 24206, + "nominally": 24207, + "hapoel": 24208, + "qu": 24209, + "daggers": 24210, + "estranged": 24211, + "fabricated": 24212, + "towels": 24213, + "vinnie": 24214, + "wcw": 24215, + "misunderstanding": 24216, + "anglia": 24217, + "nothin": 24218, + "unmistakable": 24219, + "##dust": 24220, + "##lova": 24221, + "chilly": 24222, + "marquette": 24223, + "truss": 24224, + "##edge": 24225, + "##erine": 24226, + "reece": 24227, + "##lty": 24228, + "##chemist": 24229, + "##connected": 24230, + "272": 24231, + "308": 24232, + "41st": 24233, + "bash": 24234, + "raion": 24235, + "waterfalls": 24236, + "##ump": 24237, + "##main": 24238, + "labyrinth": 24239, + "queue": 24240, + "theorist": 24241, + "##istle": 24242, + "bharatiya": 24243, + "flexed": 24244, + "soundtracks": 24245, + "rooney": 24246, + "leftist": 24247, + "patrolling": 24248, + "wharton": 24249, + "plainly": 24250, + "alleviate": 24251, + "eastman": 24252, + "schuster": 24253, + "topographic": 24254, + "engages": 24255, + "immensely": 24256, + "unbearable": 24257, + "fairchild": 24258, + "1620": 24259, + "dona": 24260, + "lurking": 24261, + "parisian": 24262, + "oliveira": 24263, + "ia": 24264, + "indictment": 24265, + "hahn": 24266, + "bangladeshi": 24267, + "##aster": 24268, + "vivo": 24269, + "##uming": 24270, + "##ential": 24271, + "antonia": 24272, + "expects": 24273, + "indoors": 24274, + "kildare": 24275, + "harlan": 24276, + "##logue": 24277, + "##ogenic": 24278, + "##sities": 24279, + "forgiven": 24280, + "##wat": 24281, + "childish": 24282, + "tavi": 24283, + "##mide": 24284, + "##orra": 24285, + "plausible": 24286, + "grimm": 24287, + "successively": 24288, + "scooted": 24289, + "##bola": 24290, + "##dget": 24291, + "##rith": 24292, + "spartans": 24293, + "emery": 24294, + "flatly": 24295, + "azure": 24296, + "epilogue": 24297, + "##wark": 24298, + "flourish": 24299, + "##iny": 24300, + "##tracted": 24301, + "##overs": 24302, + "##oshi": 24303, + "bestseller": 24304, + "distressed": 24305, + "receipt": 24306, + "spitting": 24307, + "hermit": 24308, + "topological": 24309, + "##cot": 24310, + "drilled": 24311, + "subunit": 24312, + "francs": 24313, + "##layer": 24314, + "eel": 24315, + "##fk": 24316, + "##itas": 24317, + "octopus": 24318, + "footprint": 24319, + "petitions": 24320, + "ufo": 24321, + "##say": 24322, + "##foil": 24323, + "interfering": 24324, + "leaking": 24325, + "palo": 24326, + "##metry": 24327, + "thistle": 24328, + "valiant": 24329, + "##pic": 24330, + "narayan": 24331, + "mcpherson": 24332, + "##fast": 24333, + "gonzales": 24334, + "##ym": 24335, + "##enne": 24336, + "dustin": 24337, + "novgorod": 24338, + "solos": 24339, + "##zman": 24340, + "doin": 24341, + "##raph": 24342, + "##patient": 24343, + "##meyer": 24344, + "soluble": 24345, + "ashland": 24346, + "cuffs": 24347, + "carole": 24348, + "pendleton": 24349, + "whistling": 24350, + "vassal": 24351, + "##river": 24352, + "deviation": 24353, + "revisited": 24354, + "constituents": 24355, + "rallied": 24356, + "rotate": 24357, + "loomed": 24358, + "##eil": 24359, + "##nting": 24360, + "amateurs": 24361, + "augsburg": 24362, + "auschwitz": 24363, + "crowns": 24364, + "skeletons": 24365, + "##cona": 24366, + "bonnet": 24367, + "257": 24368, + "dummy": 24369, + "globalization": 24370, + "simeon": 24371, + "sleeper": 24372, + "mandal": 24373, + "differentiated": 24374, + "##crow": 24375, + "##mare": 24376, + "milne": 24377, + "bundled": 24378, + "exasperated": 24379, + "talmud": 24380, + "owes": 24381, + "segregated": 24382, + "##feng": 24383, + "##uary": 24384, + "dentist": 24385, + "piracy": 24386, + "props": 24387, + "##rang": 24388, + "devlin": 24389, + "##torium": 24390, + "malicious": 24391, + "paws": 24392, + "##laid": 24393, + "dependency": 24394, + "##ergy": 24395, + "##fers": 24396, + "##enna": 24397, + "258": 24398, + "pistons": 24399, + "rourke": 24400, + "jed": 24401, + "grammatical": 24402, + "tres": 24403, + "maha": 24404, + "wig": 24405, + "512": 24406, + "ghostly": 24407, + "jayne": 24408, + "##achal": 24409, + "##creen": 24410, + "##ilis": 24411, + "##lins": 24412, + "##rence": 24413, + "designate": 24414, + "##with": 24415, + "arrogance": 24416, + "cambodian": 24417, + "clones": 24418, + "showdown": 24419, + "throttle": 24420, + "twain": 24421, + "##ception": 24422, + "lobes": 24423, + "metz": 24424, + "nagoya": 24425, + "335": 24426, + "braking": 24427, + "##furt": 24428, + "385": 24429, + "roaming": 24430, + "##minster": 24431, + "amin": 24432, + "crippled": 24433, + "##37": 24434, + "##llary": 24435, + "indifferent": 24436, + "hoffmann": 24437, + "idols": 24438, + "intimidating": 24439, + "1751": 24440, + "261": 24441, + "influenza": 24442, + "memo": 24443, + "onions": 24444, + "1748": 24445, + "bandage": 24446, + "consciously": 24447, + "##landa": 24448, + "##rage": 24449, + "clandestine": 24450, + "observes": 24451, + "swiped": 24452, + "tangle": 24453, + "##ener": 24454, + "##jected": 24455, + "##trum": 24456, + "##bill": 24457, + "##lta": 24458, + "hugs": 24459, + "congresses": 24460, + "josiah": 24461, + "spirited": 24462, + "##dek": 24463, + "humanist": 24464, + "managerial": 24465, + "filmmaking": 24466, + "inmate": 24467, + "rhymes": 24468, + "debuting": 24469, + "grimsby": 24470, + "ur": 24471, + "##laze": 24472, + "duplicate": 24473, + "vigor": 24474, + "##tf": 24475, + "republished": 24476, + "bolshevik": 24477, + "refurbishment": 24478, + "antibiotics": 24479, + "martini": 24480, + "methane": 24481, + "newscasts": 24482, + "royale": 24483, + "horizons": 24484, + "levant": 24485, + "iain": 24486, + "visas": 24487, + "##ischen": 24488, + "paler": 24489, + "##around": 24490, + "manifestation": 24491, + "snuck": 24492, + "alf": 24493, + "chop": 24494, + "futile": 24495, + "pedestal": 24496, + "rehab": 24497, + "##kat": 24498, + "bmg": 24499, + "kerman": 24500, + "res": 24501, + "fairbanks": 24502, + "jarrett": 24503, + "abstraction": 24504, + "saharan": 24505, + "##zek": 24506, + "1746": 24507, + "procedural": 24508, + "clearer": 24509, + "kincaid": 24510, + "sash": 24511, + "luciano": 24512, + "##ffey": 24513, + "crunch": 24514, + "helmut": 24515, + "##vara": 24516, + "revolutionaries": 24517, + "##tute": 24518, + "creamy": 24519, + "leach": 24520, + "##mmon": 24521, + "1747": 24522, + "permitting": 24523, + "nes": 24524, + "plight": 24525, + "wendell": 24526, + "##lese": 24527, + "contra": 24528, + "ts": 24529, + "clancy": 24530, + "ipa": 24531, + "mach": 24532, + "staples": 24533, + "autopsy": 24534, + "disturbances": 24535, + "nueva": 24536, + "karin": 24537, + "pontiac": 24538, + "##uding": 24539, + "proxy": 24540, + "venerable": 24541, + "haunt": 24542, + "leto": 24543, + "bergman": 24544, + "expands": 24545, + "##helm": 24546, + "wal": 24547, + "##pipe": 24548, + "canning": 24549, + "celine": 24550, + "cords": 24551, + "obesity": 24552, + "##enary": 24553, + "intrusion": 24554, + "planner": 24555, + "##phate": 24556, + "reasoned": 24557, + "sequencing": 24558, + "307": 24559, + "harrow": 24560, + "##chon": 24561, + "##dora": 24562, + "marred": 24563, + "mcintyre": 24564, + "repay": 24565, + "tarzan": 24566, + "darting": 24567, + "248": 24568, + "harrisburg": 24569, + "margarita": 24570, + "repulsed": 24571, + "##hur": 24572, + "##lding": 24573, + "belinda": 24574, + "hamburger": 24575, + "novo": 24576, + "compliant": 24577, + "runways": 24578, + "bingham": 24579, + "registrar": 24580, + "skyscraper": 24581, + "ic": 24582, + "cuthbert": 24583, + "improvisation": 24584, + "livelihood": 24585, + "##corp": 24586, + "##elial": 24587, + "admiring": 24588, + "##dened": 24589, + "sporadic": 24590, + "believer": 24591, + "casablanca": 24592, + "popcorn": 24593, + "##29": 24594, + "asha": 24595, + "shovel": 24596, + "##bek": 24597, + "##dice": 24598, + "coiled": 24599, + "tangible": 24600, + "##dez": 24601, + "casper": 24602, + "elsie": 24603, + "resin": 24604, + "tenderness": 24605, + "rectory": 24606, + "##ivision": 24607, + "avail": 24608, + "sonar": 24609, + "##mori": 24610, + "boutique": 24611, + "##dier": 24612, + "guerre": 24613, + "bathed": 24614, + "upbringing": 24615, + "vaulted": 24616, + "sandals": 24617, + "blessings": 24618, + "##naut": 24619, + "##utnant": 24620, + "1680": 24621, + "306": 24622, + "foxes": 24623, + "pia": 24624, + "corrosion": 24625, + "hesitantly": 24626, + "confederates": 24627, + "crystalline": 24628, + "footprints": 24629, + "shapiro": 24630, + "tirana": 24631, + "valentin": 24632, + "drones": 24633, + "45th": 24634, + "microscope": 24635, + "shipments": 24636, + "texted": 24637, + "inquisition": 24638, + "wry": 24639, + "guernsey": 24640, + "unauthorized": 24641, + "resigning": 24642, + "760": 24643, + "ripple": 24644, + "schubert": 24645, + "stu": 24646, + "reassure": 24647, + "felony": 24648, + "##ardo": 24649, + "brittle": 24650, + "koreans": 24651, + "##havan": 24652, + "##ives": 24653, + "dun": 24654, + "implicit": 24655, + "tyres": 24656, + "##aldi": 24657, + "##lth": 24658, + "magnolia": 24659, + "##ehan": 24660, + "##puri": 24661, + "##poulos": 24662, + "aggressively": 24663, + "fei": 24664, + "gr": 24665, + "familiarity": 24666, + "##poo": 24667, + "indicative": 24668, + "##trust": 24669, + "fundamentally": 24670, + "jimmie": 24671, + "overrun": 24672, + "395": 24673, + "anchors": 24674, + "moans": 24675, + "##opus": 24676, + "britannia": 24677, + "armagh": 24678, + "##ggle": 24679, + "purposely": 24680, + "seizing": 24681, + "##vao": 24682, + "bewildered": 24683, + "mundane": 24684, + "avoidance": 24685, + "cosmopolitan": 24686, + "geometridae": 24687, + "quartermaster": 24688, + "caf": 24689, + "415": 24690, + "chatter": 24691, + "engulfed": 24692, + "gleam": 24693, + "purge": 24694, + "##icate": 24695, + "juliette": 24696, + "jurisprudence": 24697, + "guerra": 24698, + "revisions": 24699, + "##bn": 24700, + "casimir": 24701, + "brew": 24702, + "##jm": 24703, + "1749": 24704, + "clapton": 24705, + "cloudy": 24706, + "conde": 24707, + "hermitage": 24708, + "278": 24709, + "simulations": 24710, + "torches": 24711, + "vincenzo": 24712, + "matteo": 24713, + "##rill": 24714, + "hidalgo": 24715, + "booming": 24716, + "westbound": 24717, + "accomplishment": 24718, + "tentacles": 24719, + "unaffected": 24720, + "##sius": 24721, + "annabelle": 24722, + "flopped": 24723, + "sloping": 24724, + "##litz": 24725, + "dreamer": 24726, + "interceptor": 24727, + "vu": 24728, + "##loh": 24729, + "consecration": 24730, + "copying": 24731, + "messaging": 24732, + "breaker": 24733, + "climates": 24734, + "hospitalized": 24735, + "1752": 24736, + "torino": 24737, + "afternoons": 24738, + "winfield": 24739, + "witnessing": 24740, + "##teacher": 24741, + "breakers": 24742, + "choirs": 24743, + "sawmill": 24744, + "coldly": 24745, + "##ege": 24746, + "sipping": 24747, + "haste": 24748, + "uninhabited": 24749, + "conical": 24750, + "bibliography": 24751, + "pamphlets": 24752, + "severn": 24753, + "edict": 24754, + "##oca": 24755, + "deux": 24756, + "illnesses": 24757, + "grips": 24758, + "##pl": 24759, + "rehearsals": 24760, + "sis": 24761, + "thinkers": 24762, + "tame": 24763, + "##keepers": 24764, + "1690": 24765, + "acacia": 24766, + "reformer": 24767, + "##osed": 24768, + "##rys": 24769, + "shuffling": 24770, + "##iring": 24771, + "##shima": 24772, + "eastbound": 24773, + "ionic": 24774, + "rhea": 24775, + "flees": 24776, + "littered": 24777, + "##oum": 24778, + "rocker": 24779, + "vomiting": 24780, + "groaning": 24781, + "champ": 24782, + "overwhelmingly": 24783, + "civilizations": 24784, + "paces": 24785, + "sloop": 24786, + "adoptive": 24787, + "##tish": 24788, + "skaters": 24789, + "##vres": 24790, + "aiding": 24791, + "mango": 24792, + "##joy": 24793, + "nikola": 24794, + "shriek": 24795, + "##ignon": 24796, + "pharmaceuticals": 24797, + "##mg": 24798, + "tuna": 24799, + "calvert": 24800, + "gustavo": 24801, + "stocked": 24802, + "yearbook": 24803, + "##urai": 24804, + "##mana": 24805, + "computed": 24806, + "subsp": 24807, + "riff": 24808, + "hanoi": 24809, + "kelvin": 24810, + "hamid": 24811, + "moors": 24812, + "pastures": 24813, + "summons": 24814, + "jihad": 24815, + "nectar": 24816, + "##ctors": 24817, + "bayou": 24818, + "untitled": 24819, + "pleasing": 24820, + "vastly": 24821, + "republics": 24822, + "intellect": 24823, + "##η": 24824, + "##ulio": 24825, + "##tou": 24826, + "crumbling": 24827, + "stylistic": 24828, + "sb": 24829, + "##ی": 24830, + "consolation": 24831, + "frequented": 24832, + "h₂o": 24833, + "walden": 24834, + "widows": 24835, + "##iens": 24836, + "404": 24837, + "##ignment": 24838, + "chunks": 24839, + "improves": 24840, + "288": 24841, + "grit": 24842, + "recited": 24843, + "##dev": 24844, + "snarl": 24845, + "sociological": 24846, + "##arte": 24847, + "##gul": 24848, + "inquired": 24849, + "##held": 24850, + "bruise": 24851, + "clube": 24852, + "consultancy": 24853, + "homogeneous": 24854, + "hornets": 24855, + "multiplication": 24856, + "pasta": 24857, + "prick": 24858, + "savior": 24859, + "##grin": 24860, + "##kou": 24861, + "##phile": 24862, + "yoon": 24863, + "##gara": 24864, + "grimes": 24865, + "vanishing": 24866, + "cheering": 24867, + "reacting": 24868, + "bn": 24869, + "distillery": 24870, + "##quisite": 24871, + "##vity": 24872, + "coe": 24873, + "dockyard": 24874, + "massif": 24875, + "##jord": 24876, + "escorts": 24877, + "voss": 24878, + "##valent": 24879, + "byte": 24880, + "chopped": 24881, + "hawke": 24882, + "illusions": 24883, + "workings": 24884, + "floats": 24885, + "##koto": 24886, + "##vac": 24887, + "kv": 24888, + "annapolis": 24889, + "madden": 24890, + "##onus": 24891, + "alvaro": 24892, + "noctuidae": 24893, + "##cum": 24894, + "##scopic": 24895, + "avenge": 24896, + "steamboat": 24897, + "forte": 24898, + "illustrates": 24899, + "erika": 24900, + "##trip": 24901, + "570": 24902, + "dew": 24903, + "nationalities": 24904, + "bran": 24905, + "manifested": 24906, + "thirsty": 24907, + "diversified": 24908, + "muscled": 24909, + "reborn": 24910, + "##standing": 24911, + "arson": 24912, + "##lessness": 24913, + "##dran": 24914, + "##logram": 24915, + "##boys": 24916, + "##kushima": 24917, + "##vious": 24918, + "willoughby": 24919, + "##phobia": 24920, + "286": 24921, + "alsace": 24922, + "dashboard": 24923, + "yuki": 24924, + "##chai": 24925, + "granville": 24926, + "myspace": 24927, + "publicized": 24928, + "tricked": 24929, + "##gang": 24930, + "adjective": 24931, + "##ater": 24932, + "relic": 24933, + "reorganisation": 24934, + "enthusiastically": 24935, + "indications": 24936, + "saxe": 24937, + "##lassified": 24938, + "consolidate": 24939, + "iec": 24940, + "padua": 24941, + "helplessly": 24942, + "ramps": 24943, + "renaming": 24944, + "regulars": 24945, + "pedestrians": 24946, + "accents": 24947, + "convicts": 24948, + "inaccurate": 24949, + "lowers": 24950, + "mana": 24951, + "##pati": 24952, + "barrie": 24953, + "bjp": 24954, + "outta": 24955, + "someplace": 24956, + "berwick": 24957, + "flanking": 24958, + "invoked": 24959, + "marrow": 24960, + "sparsely": 24961, + "excerpts": 24962, + "clothed": 24963, + "rei": 24964, + "##ginal": 24965, + "wept": 24966, + "##straße": 24967, + "##vish": 24968, + "alexa": 24969, + "excel": 24970, + "##ptive": 24971, + "membranes": 24972, + "aquitaine": 24973, + "creeks": 24974, + "cutler": 24975, + "sheppard": 24976, + "implementations": 24977, + "ns": 24978, + "##dur": 24979, + "fragrance": 24980, + "budge": 24981, + "concordia": 24982, + "magnesium": 24983, + "marcelo": 24984, + "##antes": 24985, + "gladly": 24986, + "vibrating": 24987, + "##rral": 24988, + "##ggles": 24989, + "montrose": 24990, + "##omba": 24991, + "lew": 24992, + "seamus": 24993, + "1630": 24994, + "cocky": 24995, + "##ament": 24996, + "##uen": 24997, + "bjorn": 24998, + "##rrick": 24999, + "fielder": 25000, + "fluttering": 25001, + "##lase": 25002, + "methyl": 25003, + "kimberley": 25004, + "mcdowell": 25005, + "reductions": 25006, + "barbed": 25007, + "##jic": 25008, + "##tonic": 25009, + "aeronautical": 25010, + "condensed": 25011, + "distracting": 25012, + "##promising": 25013, + "huffed": 25014, + "##cala": 25015, + "##sle": 25016, + "claudius": 25017, + "invincible": 25018, + "missy": 25019, + "pious": 25020, + "balthazar": 25021, + "ci": 25022, + "##lang": 25023, + "butte": 25024, + "combo": 25025, + "orson": 25026, + "##dication": 25027, + "myriad": 25028, + "1707": 25029, + "silenced": 25030, + "##fed": 25031, + "##rh": 25032, + "coco": 25033, + "netball": 25034, + "yourselves": 25035, + "##oza": 25036, + "clarify": 25037, + "heller": 25038, + "peg": 25039, + "durban": 25040, + "etudes": 25041, + "offender": 25042, + "roast": 25043, + "blackmail": 25044, + "curvature": 25045, + "##woods": 25046, + "vile": 25047, + "309": 25048, + "illicit": 25049, + "suriname": 25050, + "##linson": 25051, + "overture": 25052, + "1685": 25053, + "bubbling": 25054, + "gymnast": 25055, + "tucking": 25056, + "##mming": 25057, + "##ouin": 25058, + "maldives": 25059, + "##bala": 25060, + "gurney": 25061, + "##dda": 25062, + "##eased": 25063, + "##oides": 25064, + "backside": 25065, + "pinto": 25066, + "jars": 25067, + "racehorse": 25068, + "tending": 25069, + "##rdial": 25070, + "baronetcy": 25071, + "wiener": 25072, + "duly": 25073, + "##rke": 25074, + "barbarian": 25075, + "cupping": 25076, + "flawed": 25077, + "##thesis": 25078, + "bertha": 25079, + "pleistocene": 25080, + "puddle": 25081, + "swearing": 25082, + "##nob": 25083, + "##tically": 25084, + "fleeting": 25085, + "prostate": 25086, + "amulet": 25087, + "educating": 25088, + "##mined": 25089, + "##iti": 25090, + "##tler": 25091, + "75th": 25092, + "jens": 25093, + "respondents": 25094, + "analytics": 25095, + "cavaliers": 25096, + "papacy": 25097, + "raju": 25098, + "##iente": 25099, + "##ulum": 25100, + "##tip": 25101, + "funnel": 25102, + "271": 25103, + "disneyland": 25104, + "##lley": 25105, + "sociologist": 25106, + "##iam": 25107, + "2500": 25108, + "faulkner": 25109, + "louvre": 25110, + "menon": 25111, + "##dson": 25112, + "276": 25113, + "##ower": 25114, + "afterlife": 25115, + "mannheim": 25116, + "peptide": 25117, + "referees": 25118, + "comedians": 25119, + "meaningless": 25120, + "##anger": 25121, + "##laise": 25122, + "fabrics": 25123, + "hurley": 25124, + "renal": 25125, + "sleeps": 25126, + "##bour": 25127, + "##icle": 25128, + "breakout": 25129, + "kristin": 25130, + "roadside": 25131, + "animator": 25132, + "clover": 25133, + "disdain": 25134, + "unsafe": 25135, + "redesign": 25136, + "##urity": 25137, + "firth": 25138, + "barnsley": 25139, + "portage": 25140, + "reset": 25141, + "narrows": 25142, + "268": 25143, + "commandos": 25144, + "expansive": 25145, + "speechless": 25146, + "tubular": 25147, + "##lux": 25148, + "essendon": 25149, + "eyelashes": 25150, + "smashwords": 25151, + "##yad": 25152, + "##bang": 25153, + "##claim": 25154, + "craved": 25155, + "sprinted": 25156, + "chet": 25157, + "somme": 25158, + "astor": 25159, + "wrocław": 25160, + "orton": 25161, + "266": 25162, + "bane": 25163, + "##erving": 25164, + "##uing": 25165, + "mischief": 25166, + "##amps": 25167, + "##sund": 25168, + "scaling": 25169, + "terre": 25170, + "##xious": 25171, + "impairment": 25172, + "offenses": 25173, + "undermine": 25174, + "moi": 25175, + "soy": 25176, + "contiguous": 25177, + "arcadia": 25178, + "inuit": 25179, + "seam": 25180, + "##tops": 25181, + "macbeth": 25182, + "rebelled": 25183, + "##icative": 25184, + "##iot": 25185, + "590": 25186, + "elaborated": 25187, + "frs": 25188, + "uniformed": 25189, + "##dberg": 25190, + "259": 25191, + "powerless": 25192, + "priscilla": 25193, + "stimulated": 25194, + "980": 25195, + "qc": 25196, + "arboretum": 25197, + "frustrating": 25198, + "trieste": 25199, + "bullock": 25200, + "##nified": 25201, + "enriched": 25202, + "glistening": 25203, + "intern": 25204, + "##adia": 25205, + "locus": 25206, + "nouvelle": 25207, + "ollie": 25208, + "ike": 25209, + "lash": 25210, + "starboard": 25211, + "ee": 25212, + "tapestry": 25213, + "headlined": 25214, + "hove": 25215, + "rigged": 25216, + "##vite": 25217, + "pollock": 25218, + "##yme": 25219, + "thrive": 25220, + "clustered": 25221, + "cas": 25222, + "roi": 25223, + "gleamed": 25224, + "olympiad": 25225, + "##lino": 25226, + "pressured": 25227, + "regimes": 25228, + "##hosis": 25229, + "##lick": 25230, + "ripley": 25231, + "##ophone": 25232, + "kickoff": 25233, + "gallon": 25234, + "rockwell": 25235, + "##arable": 25236, + "crusader": 25237, + "glue": 25238, + "revolutions": 25239, + "scrambling": 25240, + "1714": 25241, + "grover": 25242, + "##jure": 25243, + "englishman": 25244, + "aztec": 25245, + "263": 25246, + "contemplating": 25247, + "coven": 25248, + "ipad": 25249, + "preach": 25250, + "triumphant": 25251, + "tufts": 25252, + "##esian": 25253, + "rotational": 25254, + "##phus": 25255, + "328": 25256, + "falkland": 25257, + "##brates": 25258, + "strewn": 25259, + "clarissa": 25260, + "rejoin": 25261, + "environmentally": 25262, + "glint": 25263, + "banded": 25264, + "drenched": 25265, + "moat": 25266, + "albanians": 25267, + "johor": 25268, + "rr": 25269, + "maestro": 25270, + "malley": 25271, + "nouveau": 25272, + "shaded": 25273, + "taxonomy": 25274, + "v6": 25275, + "adhere": 25276, + "bunk": 25277, + "airfields": 25278, + "##ritan": 25279, + "1741": 25280, + "encompass": 25281, + "remington": 25282, + "tran": 25283, + "##erative": 25284, + "amelie": 25285, + "mazda": 25286, + "friar": 25287, + "morals": 25288, + "passions": 25289, + "##zai": 25290, + "breadth": 25291, + "vis": 25292, + "##hae": 25293, + "argus": 25294, + "burnham": 25295, + "caressing": 25296, + "insider": 25297, + "rudd": 25298, + "##imov": 25299, + "##mini": 25300, + "##rso": 25301, + "italianate": 25302, + "murderous": 25303, + "textual": 25304, + "wainwright": 25305, + "armada": 25306, + "bam": 25307, + "weave": 25308, + "timer": 25309, + "##taken": 25310, + "##nh": 25311, + "fra": 25312, + "##crest": 25313, + "ardent": 25314, + "salazar": 25315, + "taps": 25316, + "tunis": 25317, + "##ntino": 25318, + "allegro": 25319, + "gland": 25320, + "philanthropic": 25321, + "##chester": 25322, + "implication": 25323, + "##optera": 25324, + "esq": 25325, + "judas": 25326, + "noticeably": 25327, + "wynn": 25328, + "##dara": 25329, + "inched": 25330, + "indexed": 25331, + "crises": 25332, + "villiers": 25333, + "bandit": 25334, + "royalties": 25335, + "patterned": 25336, + "cupboard": 25337, + "interspersed": 25338, + "accessory": 25339, + "isla": 25340, + "kendrick": 25341, + "entourage": 25342, + "stitches": 25343, + "##esthesia": 25344, + "headwaters": 25345, + "##ior": 25346, + "interlude": 25347, + "distraught": 25348, + "draught": 25349, + "1727": 25350, + "##basket": 25351, + "biased": 25352, + "sy": 25353, + "transient": 25354, + "triad": 25355, + "subgenus": 25356, + "adapting": 25357, + "kidd": 25358, + "shortstop": 25359, + "##umatic": 25360, + "dimly": 25361, + "spiked": 25362, + "mcleod": 25363, + "reprint": 25364, + "nellie": 25365, + "pretoria": 25366, + "windmill": 25367, + "##cek": 25368, + "singled": 25369, + "##mps": 25370, + "273": 25371, + "reunite": 25372, + "##orous": 25373, + "747": 25374, + "bankers": 25375, + "outlying": 25376, + "##omp": 25377, + "##ports": 25378, + "##tream": 25379, + "apologies": 25380, + "cosmetics": 25381, + "patsy": 25382, + "##deh": 25383, + "##ocks": 25384, + "##yson": 25385, + "bender": 25386, + "nantes": 25387, + "serene": 25388, + "##nad": 25389, + "lucha": 25390, + "mmm": 25391, + "323": 25392, + "##cius": 25393, + "##gli": 25394, + "cmll": 25395, + "coinage": 25396, + "nestor": 25397, + "juarez": 25398, + "##rook": 25399, + "smeared": 25400, + "sprayed": 25401, + "twitching": 25402, + "sterile": 25403, + "irina": 25404, + "embodied": 25405, + "juveniles": 25406, + "enveloped": 25407, + "miscellaneous": 25408, + "cancers": 25409, + "dq": 25410, + "gulped": 25411, + "luisa": 25412, + "crested": 25413, + "swat": 25414, + "donegal": 25415, + "ref": 25416, + "##anov": 25417, + "##acker": 25418, + "hearst": 25419, + "mercantile": 25420, + "##lika": 25421, + "doorbell": 25422, + "ua": 25423, + "vicki": 25424, + "##alla": 25425, + "##som": 25426, + "bilbao": 25427, + "psychologists": 25428, + "stryker": 25429, + "sw": 25430, + "horsemen": 25431, + "turkmenistan": 25432, + "wits": 25433, + "##national": 25434, + "anson": 25435, + "mathew": 25436, + "screenings": 25437, + "##umb": 25438, + "rihanna": 25439, + "##agne": 25440, + "##nessy": 25441, + "aisles": 25442, + "##iani": 25443, + "##osphere": 25444, + "hines": 25445, + "kenton": 25446, + "saskatoon": 25447, + "tasha": 25448, + "truncated": 25449, + "##champ": 25450, + "##itan": 25451, + "mildred": 25452, + "advises": 25453, + "fredrik": 25454, + "interpreting": 25455, + "inhibitors": 25456, + "##athi": 25457, + "spectroscopy": 25458, + "##hab": 25459, + "##kong": 25460, + "karim": 25461, + "panda": 25462, + "##oia": 25463, + "##nail": 25464, + "##vc": 25465, + "conqueror": 25466, + "kgb": 25467, + "leukemia": 25468, + "##dity": 25469, + "arrivals": 25470, + "cheered": 25471, + "pisa": 25472, + "phosphorus": 25473, + "shielded": 25474, + "##riated": 25475, + "mammal": 25476, + "unitarian": 25477, + "urgently": 25478, + "chopin": 25479, + "sanitary": 25480, + "##mission": 25481, + "spicy": 25482, + "drugged": 25483, + "hinges": 25484, + "##tort": 25485, + "tipping": 25486, + "trier": 25487, + "impoverished": 25488, + "westchester": 25489, + "##caster": 25490, + "267": 25491, + "epoch": 25492, + "nonstop": 25493, + "##gman": 25494, + "##khov": 25495, + "aromatic": 25496, + "centrally": 25497, + "cerro": 25498, + "##tively": 25499, + "##vio": 25500, + "billions": 25501, + "modulation": 25502, + "sedimentary": 25503, + "283": 25504, + "facilitating": 25505, + "outrageous": 25506, + "goldstein": 25507, + "##eak": 25508, + "##kt": 25509, + "ld": 25510, + "maitland": 25511, + "penultimate": 25512, + "pollard": 25513, + "##dance": 25514, + "fleets": 25515, + "spaceship": 25516, + "vertebrae": 25517, + "##nig": 25518, + "alcoholism": 25519, + "als": 25520, + "recital": 25521, + "##bham": 25522, + "##ference": 25523, + "##omics": 25524, + "m2": 25525, + "##bm": 25526, + "trois": 25527, + "##tropical": 25528, + "##в": 25529, + "commemorates": 25530, + "##meric": 25531, + "marge": 25532, + "##raction": 25533, + "1643": 25534, + "670": 25535, + "cosmetic": 25536, + "ravaged": 25537, + "##ige": 25538, + "catastrophe": 25539, + "eng": 25540, + "##shida": 25541, + "albrecht": 25542, + "arterial": 25543, + "bellamy": 25544, + "decor": 25545, + "harmon": 25546, + "##rde": 25547, + "bulbs": 25548, + "synchronized": 25549, + "vito": 25550, + "easiest": 25551, + "shetland": 25552, + "shielding": 25553, + "wnba": 25554, + "##glers": 25555, + "##ssar": 25556, + "##riam": 25557, + "brianna": 25558, + "cumbria": 25559, + "##aceous": 25560, + "##rard": 25561, + "cores": 25562, + "thayer": 25563, + "##nsk": 25564, + "brood": 25565, + "hilltop": 25566, + "luminous": 25567, + "carts": 25568, + "keynote": 25569, + "larkin": 25570, + "logos": 25571, + "##cta": 25572, + "##ا": 25573, + "##mund": 25574, + "##quay": 25575, + "lilith": 25576, + "tinted": 25577, + "277": 25578, + "wrestle": 25579, + "mobilization": 25580, + "##uses": 25581, + "sequential": 25582, + "siam": 25583, + "bloomfield": 25584, + "takahashi": 25585, + "274": 25586, + "##ieving": 25587, + "presenters": 25588, + "ringo": 25589, + "blazed": 25590, + "witty": 25591, + "##oven": 25592, + "##ignant": 25593, + "devastation": 25594, + "haydn": 25595, + "harmed": 25596, + "newt": 25597, + "therese": 25598, + "##peed": 25599, + "gershwin": 25600, + "molina": 25601, + "rabbis": 25602, + "sudanese": 25603, + "001": 25604, + "innate": 25605, + "restarted": 25606, + "##sack": 25607, + "##fus": 25608, + "slices": 25609, + "wb": 25610, + "##shah": 25611, + "enroll": 25612, + "hypothetical": 25613, + "hysterical": 25614, + "1743": 25615, + "fabio": 25616, + "indefinite": 25617, + "warped": 25618, + "##hg": 25619, + "exchanging": 25620, + "525": 25621, + "unsuitable": 25622, + "##sboro": 25623, + "gallo": 25624, + "1603": 25625, + "bret": 25626, + "cobalt": 25627, + "homemade": 25628, + "##hunter": 25629, + "mx": 25630, + "operatives": 25631, + "##dhar": 25632, + "terraces": 25633, + "durable": 25634, + "latch": 25635, + "pens": 25636, + "whorls": 25637, + "##ctuated": 25638, + "##eaux": 25639, + "billing": 25640, + "ligament": 25641, + "succumbed": 25642, + "##gly": 25643, + "regulators": 25644, + "spawn": 25645, + "##brick": 25646, + "##stead": 25647, + "filmfare": 25648, + "rochelle": 25649, + "##nzo": 25650, + "1725": 25651, + "circumstance": 25652, + "saber": 25653, + "supplements": 25654, + "##nsky": 25655, + "##tson": 25656, + "crowe": 25657, + "wellesley": 25658, + "carrot": 25659, + "##9th": 25660, + "##movable": 25661, + "primate": 25662, + "drury": 25663, + "sincerely": 25664, + "topical": 25665, + "##mad": 25666, + "##rao": 25667, + "callahan": 25668, + "kyiv": 25669, + "smarter": 25670, + "tits": 25671, + "undo": 25672, + "##yeh": 25673, + "announcements": 25674, + "anthologies": 25675, + "barrio": 25676, + "nebula": 25677, + "##islaus": 25678, + "##shaft": 25679, + "##tyn": 25680, + "bodyguards": 25681, + "2021": 25682, + "assassinate": 25683, + "barns": 25684, + "emmett": 25685, + "scully": 25686, + "##mah": 25687, + "##yd": 25688, + "##eland": 25689, + "##tino": 25690, + "##itarian": 25691, + "demoted": 25692, + "gorman": 25693, + "lashed": 25694, + "prized": 25695, + "adventist": 25696, + "writ": 25697, + "##gui": 25698, + "alla": 25699, + "invertebrates": 25700, + "##ausen": 25701, + "1641": 25702, + "amman": 25703, + "1742": 25704, + "align": 25705, + "healy": 25706, + "redistribution": 25707, + "##gf": 25708, + "##rize": 25709, + "insulation": 25710, + "##drop": 25711, + "adherents": 25712, + "hezbollah": 25713, + "vitro": 25714, + "ferns": 25715, + "yanking": 25716, + "269": 25717, + "php": 25718, + "registering": 25719, + "uppsala": 25720, + "cheerleading": 25721, + "confines": 25722, + "mischievous": 25723, + "tully": 25724, + "##ross": 25725, + "49th": 25726, + "docked": 25727, + "roam": 25728, + "stipulated": 25729, + "pumpkin": 25730, + "##bry": 25731, + "prompt": 25732, + "##ezer": 25733, + "blindly": 25734, + "shuddering": 25735, + "craftsmen": 25736, + "frail": 25737, + "scented": 25738, + "katharine": 25739, + "scramble": 25740, + "shaggy": 25741, + "sponge": 25742, + "helix": 25743, + "zaragoza": 25744, + "279": 25745, + "##52": 25746, + "43rd": 25747, + "backlash": 25748, + "fontaine": 25749, + "seizures": 25750, + "posse": 25751, + "cowan": 25752, + "nonfiction": 25753, + "telenovela": 25754, + "wwii": 25755, + "hammered": 25756, + "undone": 25757, + "##gpur": 25758, + "encircled": 25759, + "irs": 25760, + "##ivation": 25761, + "artefacts": 25762, + "oneself": 25763, + "searing": 25764, + "smallpox": 25765, + "##belle": 25766, + "##osaurus": 25767, + "shandong": 25768, + "breached": 25769, + "upland": 25770, + "blushing": 25771, + "rankin": 25772, + "infinitely": 25773, + "psyche": 25774, + "tolerated": 25775, + "docking": 25776, + "evicted": 25777, + "##col": 25778, + "unmarked": 25779, + "##lving": 25780, + "gnome": 25781, + "lettering": 25782, + "litres": 25783, + "musique": 25784, + "##oint": 25785, + "benevolent": 25786, + "##jal": 25787, + "blackened": 25788, + "##anna": 25789, + "mccall": 25790, + "racers": 25791, + "tingle": 25792, + "##ocene": 25793, + "##orestation": 25794, + "introductions": 25795, + "radically": 25796, + "292": 25797, + "##hiff": 25798, + "##باد": 25799, + "1610": 25800, + "1739": 25801, + "munchen": 25802, + "plead": 25803, + "##nka": 25804, + "condo": 25805, + "scissors": 25806, + "##sight": 25807, + "##tens": 25808, + "apprehension": 25809, + "##cey": 25810, + "##yin": 25811, + "hallmark": 25812, + "watering": 25813, + "formulas": 25814, + "sequels": 25815, + "##llas": 25816, + "aggravated": 25817, + "bae": 25818, + "commencing": 25819, + "##building": 25820, + "enfield": 25821, + "prohibits": 25822, + "marne": 25823, + "vedic": 25824, + "civilized": 25825, + "euclidean": 25826, + "jagger": 25827, + "beforehand": 25828, + "blasts": 25829, + "dumont": 25830, + "##arney": 25831, + "##nem": 25832, + "740": 25833, + "conversions": 25834, + "hierarchical": 25835, + "rios": 25836, + "simulator": 25837, + "##dya": 25838, + "##lellan": 25839, + "hedges": 25840, + "oleg": 25841, + "thrusts": 25842, + "shadowed": 25843, + "darby": 25844, + "maximize": 25845, + "1744": 25846, + "gregorian": 25847, + "##nded": 25848, + "##routed": 25849, + "sham": 25850, + "unspecified": 25851, + "##hog": 25852, + "emory": 25853, + "factual": 25854, + "##smo": 25855, + "##tp": 25856, + "fooled": 25857, + "##rger": 25858, + "ortega": 25859, + "wellness": 25860, + "marlon": 25861, + "##oton": 25862, + "##urance": 25863, + "casket": 25864, + "keating": 25865, + "ley": 25866, + "enclave": 25867, + "##ayan": 25868, + "char": 25869, + "influencing": 25870, + "jia": 25871, + "##chenko": 25872, + "412": 25873, + "ammonia": 25874, + "erebidae": 25875, + "incompatible": 25876, + "violins": 25877, + "cornered": 25878, + "##arat": 25879, + "grooves": 25880, + "astronauts": 25881, + "columbian": 25882, + "rampant": 25883, + "fabrication": 25884, + "kyushu": 25885, + "mahmud": 25886, + "vanish": 25887, + "##dern": 25888, + "mesopotamia": 25889, + "##lete": 25890, + "ict": 25891, + "##rgen": 25892, + "caspian": 25893, + "kenji": 25894, + "pitted": 25895, + "##vered": 25896, + "999": 25897, + "grimace": 25898, + "roanoke": 25899, + "tchaikovsky": 25900, + "twinned": 25901, + "##analysis": 25902, + "##awan": 25903, + "xinjiang": 25904, + "arias": 25905, + "clemson": 25906, + "kazakh": 25907, + "sizable": 25908, + "1662": 25909, + "##khand": 25910, + "##vard": 25911, + "plunge": 25912, + "tatum": 25913, + "vittorio": 25914, + "##nden": 25915, + "cholera": 25916, + "##dana": 25917, + "##oper": 25918, + "bracing": 25919, + "indifference": 25920, + "projectile": 25921, + "superliga": 25922, + "##chee": 25923, + "realises": 25924, + "upgrading": 25925, + "299": 25926, + "porte": 25927, + "retribution": 25928, + "##vies": 25929, + "nk": 25930, + "stil": 25931, + "##resses": 25932, + "ama": 25933, + "bureaucracy": 25934, + "blackberry": 25935, + "bosch": 25936, + "testosterone": 25937, + "collapses": 25938, + "greer": 25939, + "##pathic": 25940, + "ioc": 25941, + "fifties": 25942, + "malls": 25943, + "##erved": 25944, + "bao": 25945, + "baskets": 25946, + "adolescents": 25947, + "siegfried": 25948, + "##osity": 25949, + "##tosis": 25950, + "mantra": 25951, + "detecting": 25952, + "existent": 25953, + "fledgling": 25954, + "##cchi": 25955, + "dissatisfied": 25956, + "gan": 25957, + "telecommunication": 25958, + "mingled": 25959, + "sobbed": 25960, + "6000": 25961, + "controversies": 25962, + "outdated": 25963, + "taxis": 25964, + "##raus": 25965, + "fright": 25966, + "slams": 25967, + "##lham": 25968, + "##fect": 25969, + "##tten": 25970, + "detectors": 25971, + "fetal": 25972, + "tanned": 25973, + "##uw": 25974, + "fray": 25975, + "goth": 25976, + "olympian": 25977, + "skipping": 25978, + "mandates": 25979, + "scratches": 25980, + "sheng": 25981, + "unspoken": 25982, + "hyundai": 25983, + "tracey": 25984, + "hotspur": 25985, + "restrictive": 25986, + "##buch": 25987, + "americana": 25988, + "mundo": 25989, + "##bari": 25990, + "burroughs": 25991, + "diva": 25992, + "vulcan": 25993, + "##6th": 25994, + "distinctions": 25995, + "thumping": 25996, + "##ngen": 25997, + "mikey": 25998, + "sheds": 25999, + "fide": 26000, + "rescues": 26001, + "springsteen": 26002, + "vested": 26003, + "valuation": 26004, + "##ece": 26005, + "##ely": 26006, + "pinnacle": 26007, + "rake": 26008, + "sylvie": 26009, + "##edo": 26010, + "almond": 26011, + "quivering": 26012, + "##irus": 26013, + "alteration": 26014, + "faltered": 26015, + "##wad": 26016, + "51st": 26017, + "hydra": 26018, + "ticked": 26019, + "##kato": 26020, + "recommends": 26021, + "##dicated": 26022, + "antigua": 26023, + "arjun": 26024, + "stagecoach": 26025, + "wilfred": 26026, + "trickle": 26027, + "pronouns": 26028, + "##pon": 26029, + "aryan": 26030, + "nighttime": 26031, + "##anian": 26032, + "gall": 26033, + "pea": 26034, + "stitch": 26035, + "##hei": 26036, + "leung": 26037, + "milos": 26038, + "##dini": 26039, + "eritrea": 26040, + "nexus": 26041, + "starved": 26042, + "snowfall": 26043, + "kant": 26044, + "parasitic": 26045, + "cot": 26046, + "discus": 26047, + "hana": 26048, + "strikers": 26049, + "appleton": 26050, + "kitchens": 26051, + "##erina": 26052, + "##partisan": 26053, + "##itha": 26054, + "##vius": 26055, + "disclose": 26056, + "metis": 26057, + "##channel": 26058, + "1701": 26059, + "tesla": 26060, + "##vera": 26061, + "fitch": 26062, + "1735": 26063, + "blooded": 26064, + "##tila": 26065, + "decimal": 26066, + "##tang": 26067, + "##bai": 26068, + "cyclones": 26069, + "eun": 26070, + "bottled": 26071, + "peas": 26072, + "pensacola": 26073, + "basha": 26074, + "bolivian": 26075, + "crabs": 26076, + "boil": 26077, + "lanterns": 26078, + "partridge": 26079, + "roofed": 26080, + "1645": 26081, + "necks": 26082, + "##phila": 26083, + "opined": 26084, + "patting": 26085, + "##kla": 26086, + "##lland": 26087, + "chuckles": 26088, + "volta": 26089, + "whereupon": 26090, + "##nche": 26091, + "devout": 26092, + "euroleague": 26093, + "suicidal": 26094, + "##dee": 26095, + "inherently": 26096, + "involuntary": 26097, + "knitting": 26098, + "nasser": 26099, + "##hide": 26100, + "puppets": 26101, + "colourful": 26102, + "courageous": 26103, + "southend": 26104, + "stills": 26105, + "miraculous": 26106, + "hodgson": 26107, + "richer": 26108, + "rochdale": 26109, + "ethernet": 26110, + "greta": 26111, + "uniting": 26112, + "prism": 26113, + "umm": 26114, + "##haya": 26115, + "##itical": 26116, + "##utation": 26117, + "deterioration": 26118, + "pointe": 26119, + "prowess": 26120, + "##ropriation": 26121, + "lids": 26122, + "scranton": 26123, + "billings": 26124, + "subcontinent": 26125, + "##koff": 26126, + "##scope": 26127, + "brute": 26128, + "kellogg": 26129, + "psalms": 26130, + "degraded": 26131, + "##vez": 26132, + "stanisław": 26133, + "##ructured": 26134, + "ferreira": 26135, + "pun": 26136, + "astonishing": 26137, + "gunnar": 26138, + "##yat": 26139, + "arya": 26140, + "prc": 26141, + "gottfried": 26142, + "##tight": 26143, + "excursion": 26144, + "##ographer": 26145, + "dina": 26146, + "##quil": 26147, + "##nare": 26148, + "huffington": 26149, + "illustrious": 26150, + "wilbur": 26151, + "gundam": 26152, + "verandah": 26153, + "##zard": 26154, + "naacp": 26155, + "##odle": 26156, + "constructive": 26157, + "fjord": 26158, + "kade": 26159, + "##naud": 26160, + "generosity": 26161, + "thrilling": 26162, + "baseline": 26163, + "cayman": 26164, + "frankish": 26165, + "plastics": 26166, + "accommodations": 26167, + "zoological": 26168, + "##fting": 26169, + "cedric": 26170, + "qb": 26171, + "motorized": 26172, + "##dome": 26173, + "##otted": 26174, + "squealed": 26175, + "tackled": 26176, + "canucks": 26177, + "budgets": 26178, + "situ": 26179, + "asthma": 26180, + "dail": 26181, + "gabled": 26182, + "grasslands": 26183, + "whimpered": 26184, + "writhing": 26185, + "judgments": 26186, + "##65": 26187, + "minnie": 26188, + "pv": 26189, + "##carbon": 26190, + "bananas": 26191, + "grille": 26192, + "domes": 26193, + "monique": 26194, + "odin": 26195, + "maguire": 26196, + "markham": 26197, + "tierney": 26198, + "##estra": 26199, + "##chua": 26200, + "libel": 26201, + "poke": 26202, + "speedy": 26203, + "atrium": 26204, + "laval": 26205, + "notwithstanding": 26206, + "##edly": 26207, + "fai": 26208, + "kala": 26209, + "##sur": 26210, + "robb": 26211, + "##sma": 26212, + "listings": 26213, + "luz": 26214, + "supplementary": 26215, + "tianjin": 26216, + "##acing": 26217, + "enzo": 26218, + "jd": 26219, + "ric": 26220, + "scanner": 26221, + "croats": 26222, + "transcribed": 26223, + "##49": 26224, + "arden": 26225, + "cv": 26226, + "##hair": 26227, + "##raphy": 26228, + "##lver": 26229, + "##uy": 26230, + "357": 26231, + "seventies": 26232, + "staggering": 26233, + "alam": 26234, + "horticultural": 26235, + "hs": 26236, + "regression": 26237, + "timbers": 26238, + "blasting": 26239, + "##ounded": 26240, + "montagu": 26241, + "manipulating": 26242, + "##cit": 26243, + "catalytic": 26244, + "1550": 26245, + "troopers": 26246, + "##meo": 26247, + "condemnation": 26248, + "fitzpatrick": 26249, + "##oire": 26250, + "##roved": 26251, + "inexperienced": 26252, + "1670": 26253, + "castes": 26254, + "##lative": 26255, + "outing": 26256, + "314": 26257, + "dubois": 26258, + "flicking": 26259, + "quarrel": 26260, + "ste": 26261, + "learners": 26262, + "1625": 26263, + "iq": 26264, + "whistled": 26265, + "##class": 26266, + "282": 26267, + "classify": 26268, + "tariffs": 26269, + "temperament": 26270, + "355": 26271, + "folly": 26272, + "liszt": 26273, + "##yles": 26274, + "immersed": 26275, + "jordanian": 26276, + "ceasefire": 26277, + "apparel": 26278, + "extras": 26279, + "maru": 26280, + "fished": 26281, + "##bio": 26282, + "harta": 26283, + "stockport": 26284, + "assortment": 26285, + "craftsman": 26286, + "paralysis": 26287, + "transmitters": 26288, + "##cola": 26289, + "blindness": 26290, + "##wk": 26291, + "fatally": 26292, + "proficiency": 26293, + "solemnly": 26294, + "##orno": 26295, + "repairing": 26296, + "amore": 26297, + "groceries": 26298, + "ultraviolet": 26299, + "##chase": 26300, + "schoolhouse": 26301, + "##tua": 26302, + "resurgence": 26303, + "nailed": 26304, + "##otype": 26305, + "##×": 26306, + "ruse": 26307, + "saliva": 26308, + "diagrams": 26309, + "##tructing": 26310, + "albans": 26311, + "rann": 26312, + "thirties": 26313, + "1b": 26314, + "antennas": 26315, + "hilarious": 26316, + "cougars": 26317, + "paddington": 26318, + "stats": 26319, + "##eger": 26320, + "breakaway": 26321, + "ipod": 26322, + "reza": 26323, + "authorship": 26324, + "prohibiting": 26325, + "scoffed": 26326, + "##etz": 26327, + "##ttle": 26328, + "conscription": 26329, + "defected": 26330, + "trondheim": 26331, + "##fires": 26332, + "ivanov": 26333, + "keenan": 26334, + "##adan": 26335, + "##ciful": 26336, + "##fb": 26337, + "##slow": 26338, + "locating": 26339, + "##ials": 26340, + "##tford": 26341, + "cadiz": 26342, + "basalt": 26343, + "blankly": 26344, + "interned": 26345, + "rags": 26346, + "rattling": 26347, + "##tick": 26348, + "carpathian": 26349, + "reassured": 26350, + "sync": 26351, + "bum": 26352, + "guildford": 26353, + "iss": 26354, + "staunch": 26355, + "##onga": 26356, + "astronomers": 26357, + "sera": 26358, + "sofie": 26359, + "emergencies": 26360, + "susquehanna": 26361, + "##heard": 26362, + "duc": 26363, + "mastery": 26364, + "vh1": 26365, + "williamsburg": 26366, + "bayer": 26367, + "buckled": 26368, + "craving": 26369, + "##khan": 26370, + "##rdes": 26371, + "bloomington": 26372, + "##write": 26373, + "alton": 26374, + "barbecue": 26375, + "##bians": 26376, + "justine": 26377, + "##hri": 26378, + "##ndt": 26379, + "delightful": 26380, + "smartphone": 26381, + "newtown": 26382, + "photon": 26383, + "retrieval": 26384, + "peugeot": 26385, + "hissing": 26386, + "##monium": 26387, + "##orough": 26388, + "flavors": 26389, + "lighted": 26390, + "relaunched": 26391, + "tainted": 26392, + "##games": 26393, + "##lysis": 26394, + "anarchy": 26395, + "microscopic": 26396, + "hopping": 26397, + "adept": 26398, + "evade": 26399, + "evie": 26400, + "##beau": 26401, + "inhibit": 26402, + "sinn": 26403, + "adjustable": 26404, + "hurst": 26405, + "intuition": 26406, + "wilton": 26407, + "cisco": 26408, + "44th": 26409, + "lawful": 26410, + "lowlands": 26411, + "stockings": 26412, + "thierry": 26413, + "##dalen": 26414, + "##hila": 26415, + "##nai": 26416, + "fates": 26417, + "prank": 26418, + "tb": 26419, + "maison": 26420, + "lobbied": 26421, + "provocative": 26422, + "1724": 26423, + "4a": 26424, + "utopia": 26425, + "##qual": 26426, + "carbonate": 26427, + "gujarati": 26428, + "purcell": 26429, + "##rford": 26430, + "curtiss": 26431, + "##mei": 26432, + "overgrown": 26433, + "arenas": 26434, + "mediation": 26435, + "swallows": 26436, + "##rnik": 26437, + "respectful": 26438, + "turnbull": 26439, + "##hedron": 26440, + "##hope": 26441, + "alyssa": 26442, + "ozone": 26443, + "##ʻi": 26444, + "ami": 26445, + "gestapo": 26446, + "johansson": 26447, + "snooker": 26448, + "canteen": 26449, + "cuff": 26450, + "declines": 26451, + "empathy": 26452, + "stigma": 26453, + "##ags": 26454, + "##iner": 26455, + "##raine": 26456, + "taxpayers": 26457, + "gui": 26458, + "volga": 26459, + "##wright": 26460, + "##copic": 26461, + "lifespan": 26462, + "overcame": 26463, + "tattooed": 26464, + "enactment": 26465, + "giggles": 26466, + "##ador": 26467, + "##camp": 26468, + "barrington": 26469, + "bribe": 26470, + "obligatory": 26471, + "orbiting": 26472, + "peng": 26473, + "##enas": 26474, + "elusive": 26475, + "sucker": 26476, + "##vating": 26477, + "cong": 26478, + "hardship": 26479, + "empowered": 26480, + "anticipating": 26481, + "estrada": 26482, + "cryptic": 26483, + "greasy": 26484, + "detainees": 26485, + "planck": 26486, + "sudbury": 26487, + "plaid": 26488, + "dod": 26489, + "marriott": 26490, + "kayla": 26491, + "##ears": 26492, + "##vb": 26493, + "##zd": 26494, + "mortally": 26495, + "##hein": 26496, + "cognition": 26497, + "radha": 26498, + "319": 26499, + "liechtenstein": 26500, + "meade": 26501, + "richly": 26502, + "argyle": 26503, + "harpsichord": 26504, + "liberalism": 26505, + "trumpets": 26506, + "lauded": 26507, + "tyrant": 26508, + "salsa": 26509, + "tiled": 26510, + "lear": 26511, + "promoters": 26512, + "reused": 26513, + "slicing": 26514, + "trident": 26515, + "##chuk": 26516, + "##gami": 26517, + "##lka": 26518, + "cantor": 26519, + "checkpoint": 26520, + "##points": 26521, + "gaul": 26522, + "leger": 26523, + "mammalian": 26524, + "##tov": 26525, + "##aar": 26526, + "##schaft": 26527, + "doha": 26528, + "frenchman": 26529, + "nirvana": 26530, + "##vino": 26531, + "delgado": 26532, + "headlining": 26533, + "##eron": 26534, + "##iography": 26535, + "jug": 26536, + "tko": 26537, + "1649": 26538, + "naga": 26539, + "intersections": 26540, + "##jia": 26541, + "benfica": 26542, + "nawab": 26543, + "##suka": 26544, + "ashford": 26545, + "gulp": 26546, + "##deck": 26547, + "##vill": 26548, + "##rug": 26549, + "brentford": 26550, + "frazier": 26551, + "pleasures": 26552, + "dunne": 26553, + "potsdam": 26554, + "shenzhen": 26555, + "dentistry": 26556, + "##tec": 26557, + "flanagan": 26558, + "##dorff": 26559, + "##hear": 26560, + "chorale": 26561, + "dinah": 26562, + "prem": 26563, + "quezon": 26564, + "##rogated": 26565, + "relinquished": 26566, + "sutra": 26567, + "terri": 26568, + "##pani": 26569, + "flaps": 26570, + "##rissa": 26571, + "poly": 26572, + "##rnet": 26573, + "homme": 26574, + "aback": 26575, + "##eki": 26576, + "linger": 26577, + "womb": 26578, + "##kson": 26579, + "##lewood": 26580, + "doorstep": 26581, + "orthodoxy": 26582, + "threaded": 26583, + "westfield": 26584, + "##rval": 26585, + "dioceses": 26586, + "fridays": 26587, + "subsided": 26588, + "##gata": 26589, + "loyalists": 26590, + "##biotic": 26591, + "##ettes": 26592, + "letterman": 26593, + "lunatic": 26594, + "prelate": 26595, + "tenderly": 26596, + "invariably": 26597, + "souza": 26598, + "thug": 26599, + "winslow": 26600, + "##otide": 26601, + "furlongs": 26602, + "gogh": 26603, + "jeopardy": 26604, + "##runa": 26605, + "pegasus": 26606, + "##umble": 26607, + "humiliated": 26608, + "standalone": 26609, + "tagged": 26610, + "##roller": 26611, + "freshmen": 26612, + "klan": 26613, + "##bright": 26614, + "attaining": 26615, + "initiating": 26616, + "transatlantic": 26617, + "logged": 26618, + "viz": 26619, + "##uance": 26620, + "1723": 26621, + "combatants": 26622, + "intervening": 26623, + "stephane": 26624, + "chieftain": 26625, + "despised": 26626, + "grazed": 26627, + "317": 26628, + "cdc": 26629, + "galveston": 26630, + "godzilla": 26631, + "macro": 26632, + "simulate": 26633, + "##planes": 26634, + "parades": 26635, + "##esses": 26636, + "960": 26637, + "##ductive": 26638, + "##unes": 26639, + "equator": 26640, + "overdose": 26641, + "##cans": 26642, + "##hosh": 26643, + "##lifting": 26644, + "joshi": 26645, + "epstein": 26646, + "sonora": 26647, + "treacherous": 26648, + "aquatics": 26649, + "manchu": 26650, + "responsive": 26651, + "##sation": 26652, + "supervisory": 26653, + "##christ": 26654, + "##llins": 26655, + "##ibar": 26656, + "##balance": 26657, + "##uso": 26658, + "kimball": 26659, + "karlsruhe": 26660, + "mab": 26661, + "##emy": 26662, + "ignores": 26663, + "phonetic": 26664, + "reuters": 26665, + "spaghetti": 26666, + "820": 26667, + "almighty": 26668, + "danzig": 26669, + "rumbling": 26670, + "tombstone": 26671, + "designations": 26672, + "lured": 26673, + "outset": 26674, + "##felt": 26675, + "supermarkets": 26676, + "##wt": 26677, + "grupo": 26678, + "kei": 26679, + "kraft": 26680, + "susanna": 26681, + "##blood": 26682, + "comprehension": 26683, + "genealogy": 26684, + "##aghan": 26685, + "##verted": 26686, + "redding": 26687, + "##ythe": 26688, + "1722": 26689, + "bowing": 26690, + "##pore": 26691, + "##roi": 26692, + "lest": 26693, + "sharpened": 26694, + "fulbright": 26695, + "valkyrie": 26696, + "sikhs": 26697, + "##unds": 26698, + "swans": 26699, + "bouquet": 26700, + "merritt": 26701, + "##tage": 26702, + "##venting": 26703, + "commuted": 26704, + "redhead": 26705, + "clerks": 26706, + "leasing": 26707, + "cesare": 26708, + "dea": 26709, + "hazy": 26710, + "##vances": 26711, + "fledged": 26712, + "greenfield": 26713, + "servicemen": 26714, + "##gical": 26715, + "armando": 26716, + "blackout": 26717, + "dt": 26718, + "sagged": 26719, + "downloadable": 26720, + "intra": 26721, + "potion": 26722, + "pods": 26723, + "##4th": 26724, + "##mism": 26725, + "xp": 26726, + "attendants": 26727, + "gambia": 26728, + "stale": 26729, + "##ntine": 26730, + "plump": 26731, + "asteroids": 26732, + "rediscovered": 26733, + "buds": 26734, + "flea": 26735, + "hive": 26736, + "##neas": 26737, + "1737": 26738, + "classifications": 26739, + "debuts": 26740, + "##eles": 26741, + "olympus": 26742, + "scala": 26743, + "##eurs": 26744, + "##gno": 26745, + "##mute": 26746, + "hummed": 26747, + "sigismund": 26748, + "visuals": 26749, + "wiggled": 26750, + "await": 26751, + "pilasters": 26752, + "clench": 26753, + "sulfate": 26754, + "##ances": 26755, + "bellevue": 26756, + "enigma": 26757, + "trainee": 26758, + "snort": 26759, + "##sw": 26760, + "clouded": 26761, + "denim": 26762, + "##rank": 26763, + "##rder": 26764, + "churning": 26765, + "hartman": 26766, + "lodges": 26767, + "riches": 26768, + "sima": 26769, + "##missible": 26770, + "accountable": 26771, + "socrates": 26772, + "regulates": 26773, + "mueller": 26774, + "##cr": 26775, + "1702": 26776, + "avoids": 26777, + "solids": 26778, + "himalayas": 26779, + "nutrient": 26780, + "pup": 26781, + "##jevic": 26782, + "squat": 26783, + "fades": 26784, + "nec": 26785, + "##lates": 26786, + "##pina": 26787, + "##rona": 26788, + "##ου": 26789, + "privateer": 26790, + "tequila": 26791, + "##gative": 26792, + "##mpton": 26793, + "apt": 26794, + "hornet": 26795, + "immortals": 26796, + "##dou": 26797, + "asturias": 26798, + "cleansing": 26799, + "dario": 26800, + "##rries": 26801, + "##anta": 26802, + "etymology": 26803, + "servicing": 26804, + "zhejiang": 26805, + "##venor": 26806, + "##nx": 26807, + "horned": 26808, + "erasmus": 26809, + "rayon": 26810, + "relocating": 26811, + "£10": 26812, + "##bags": 26813, + "escalated": 26814, + "promenade": 26815, + "stubble": 26816, + "2010s": 26817, + "artisans": 26818, + "axial": 26819, + "liquids": 26820, + "mora": 26821, + "sho": 26822, + "yoo": 26823, + "##tsky": 26824, + "bundles": 26825, + "oldies": 26826, + "##nally": 26827, + "notification": 26828, + "bastion": 26829, + "##ths": 26830, + "sparkle": 26831, + "##lved": 26832, + "1728": 26833, + "leash": 26834, + "pathogen": 26835, + "highs": 26836, + "##hmi": 26837, + "immature": 26838, + "880": 26839, + "gonzaga": 26840, + "ignatius": 26841, + "mansions": 26842, + "monterrey": 26843, + "sweets": 26844, + "bryson": 26845, + "##loe": 26846, + "polled": 26847, + "regatta": 26848, + "brightest": 26849, + "pei": 26850, + "rosy": 26851, + "squid": 26852, + "hatfield": 26853, + "payroll": 26854, + "addict": 26855, + "meath": 26856, + "cornerback": 26857, + "heaviest": 26858, + "lodging": 26859, + "##mage": 26860, + "capcom": 26861, + "rippled": 26862, + "##sily": 26863, + "barnet": 26864, + "mayhem": 26865, + "ymca": 26866, + "snuggled": 26867, + "rousseau": 26868, + "##cute": 26869, + "blanchard": 26870, + "284": 26871, + "fragmented": 26872, + "leighton": 26873, + "chromosomes": 26874, + "risking": 26875, + "##md": 26876, + "##strel": 26877, + "##utter": 26878, + "corinne": 26879, + "coyotes": 26880, + "cynical": 26881, + "hiroshi": 26882, + "yeomanry": 26883, + "##ractive": 26884, + "ebook": 26885, + "grading": 26886, + "mandela": 26887, + "plume": 26888, + "agustin": 26889, + "magdalene": 26890, + "##rkin": 26891, + "bea": 26892, + "femme": 26893, + "trafford": 26894, + "##coll": 26895, + "##lun": 26896, + "##tance": 26897, + "52nd": 26898, + "fourier": 26899, + "upton": 26900, + "##mental": 26901, + "camilla": 26902, + "gust": 26903, + "iihf": 26904, + "islamabad": 26905, + "longevity": 26906, + "##kala": 26907, + "feldman": 26908, + "netting": 26909, + "##rization": 26910, + "endeavour": 26911, + "foraging": 26912, + "mfa": 26913, + "orr": 26914, + "##open": 26915, + "greyish": 26916, + "contradiction": 26917, + "graz": 26918, + "##ruff": 26919, + "handicapped": 26920, + "marlene": 26921, + "tweed": 26922, + "oaxaca": 26923, + "spp": 26924, + "campos": 26925, + "miocene": 26926, + "pri": 26927, + "configured": 26928, + "cooks": 26929, + "pluto": 26930, + "cozy": 26931, + "pornographic": 26932, + "##entes": 26933, + "70th": 26934, + "fairness": 26935, + "glided": 26936, + "jonny": 26937, + "lynne": 26938, + "rounding": 26939, + "sired": 26940, + "##emon": 26941, + "##nist": 26942, + "remade": 26943, + "uncover": 26944, + "##mack": 26945, + "complied": 26946, + "lei": 26947, + "newsweek": 26948, + "##jured": 26949, + "##parts": 26950, + "##enting": 26951, + "##pg": 26952, + "293": 26953, + "finer": 26954, + "guerrillas": 26955, + "athenian": 26956, + "deng": 26957, + "disused": 26958, + "stepmother": 26959, + "accuse": 26960, + "gingerly": 26961, + "seduction": 26962, + "521": 26963, + "confronting": 26964, + "##walker": 26965, + "##going": 26966, + "gora": 26967, + "nostalgia": 26968, + "sabres": 26969, + "virginity": 26970, + "wrenched": 26971, + "##minated": 26972, + "syndication": 26973, + "wielding": 26974, + "eyre": 26975, + "##56": 26976, + "##gnon": 26977, + "##igny": 26978, + "behaved": 26979, + "taxpayer": 26980, + "sweeps": 26981, + "##growth": 26982, + "childless": 26983, + "gallant": 26984, + "##ywood": 26985, + "amplified": 26986, + "geraldine": 26987, + "scrape": 26988, + "##ffi": 26989, + "babylonian": 26990, + "fresco": 26991, + "##rdan": 26992, + "##kney": 26993, + "##position": 26994, + "1718": 26995, + "restricting": 26996, + "tack": 26997, + "fukuoka": 26998, + "osborn": 26999, + "selector": 27000, + "partnering": 27001, + "##dlow": 27002, + "318": 27003, + "gnu": 27004, + "kia": 27005, + "tak": 27006, + "whitley": 27007, + "gables": 27008, + "##54": 27009, + "##mania": 27010, + "mri": 27011, + "softness": 27012, + "immersion": 27013, + "##bots": 27014, + "##evsky": 27015, + "1713": 27016, + "chilling": 27017, + "insignificant": 27018, + "pcs": 27019, + "##uis": 27020, + "elites": 27021, + "lina": 27022, + "purported": 27023, + "supplemental": 27024, + "teaming": 27025, + "##americana": 27026, + "##dding": 27027, + "##inton": 27028, + "proficient": 27029, + "rouen": 27030, + "##nage": 27031, + "##rret": 27032, + "niccolo": 27033, + "selects": 27034, + "##bread": 27035, + "fluffy": 27036, + "1621": 27037, + "gruff": 27038, + "knotted": 27039, + "mukherjee": 27040, + "polgara": 27041, + "thrash": 27042, + "nicholls": 27043, + "secluded": 27044, + "smoothing": 27045, + "thru": 27046, + "corsica": 27047, + "loaf": 27048, + "whitaker": 27049, + "inquiries": 27050, + "##rrier": 27051, + "##kam": 27052, + "indochina": 27053, + "289": 27054, + "marlins": 27055, + "myles": 27056, + "peking": 27057, + "##tea": 27058, + "extracts": 27059, + "pastry": 27060, + "superhuman": 27061, + "connacht": 27062, + "vogel": 27063, + "##ditional": 27064, + "##het": 27065, + "##udged": 27066, + "##lash": 27067, + "gloss": 27068, + "quarries": 27069, + "refit": 27070, + "teaser": 27071, + "##alic": 27072, + "##gaon": 27073, + "20s": 27074, + "materialized": 27075, + "sling": 27076, + "camped": 27077, + "pickering": 27078, + "tung": 27079, + "tracker": 27080, + "pursuant": 27081, + "##cide": 27082, + "cranes": 27083, + "soc": 27084, + "##cini": 27085, + "##typical": 27086, + "##viere": 27087, + "anhalt": 27088, + "overboard": 27089, + "workout": 27090, + "chores": 27091, + "fares": 27092, + "orphaned": 27093, + "stains": 27094, + "##logie": 27095, + "fenton": 27096, + "surpassing": 27097, + "joyah": 27098, + "triggers": 27099, + "##itte": 27100, + "grandmaster": 27101, + "##lass": 27102, + "##lists": 27103, + "clapping": 27104, + "fraudulent": 27105, + "ledger": 27106, + "nagasaki": 27107, + "##cor": 27108, + "##nosis": 27109, + "##tsa": 27110, + "eucalyptus": 27111, + "tun": 27112, + "##icio": 27113, + "##rney": 27114, + "##tara": 27115, + "dax": 27116, + "heroism": 27117, + "ina": 27118, + "wrexham": 27119, + "onboard": 27120, + "unsigned": 27121, + "##dates": 27122, + "moshe": 27123, + "galley": 27124, + "winnie": 27125, + "droplets": 27126, + "exiles": 27127, + "praises": 27128, + "watered": 27129, + "noodles": 27130, + "##aia": 27131, + "fein": 27132, + "adi": 27133, + "leland": 27134, + "multicultural": 27135, + "stink": 27136, + "bingo": 27137, + "comets": 27138, + "erskine": 27139, + "modernized": 27140, + "canned": 27141, + "constraint": 27142, + "domestically": 27143, + "chemotherapy": 27144, + "featherweight": 27145, + "stifled": 27146, + "##mum": 27147, + "darkly": 27148, + "irresistible": 27149, + "refreshing": 27150, + "hasty": 27151, + "isolate": 27152, + "##oys": 27153, + "kitchener": 27154, + "planners": 27155, + "##wehr": 27156, + "cages": 27157, + "yarn": 27158, + "implant": 27159, + "toulon": 27160, + "elects": 27161, + "childbirth": 27162, + "yue": 27163, + "##lind": 27164, + "##lone": 27165, + "cn": 27166, + "rightful": 27167, + "sportsman": 27168, + "junctions": 27169, + "remodeled": 27170, + "specifies": 27171, + "##rgh": 27172, + "291": 27173, + "##oons": 27174, + "complimented": 27175, + "##urgent": 27176, + "lister": 27177, + "ot": 27178, + "##logic": 27179, + "bequeathed": 27180, + "cheekbones": 27181, + "fontana": 27182, + "gabby": 27183, + "##dial": 27184, + "amadeus": 27185, + "corrugated": 27186, + "maverick": 27187, + "resented": 27188, + "triangles": 27189, + "##hered": 27190, + "##usly": 27191, + "nazareth": 27192, + "tyrol": 27193, + "1675": 27194, + "assent": 27195, + "poorer": 27196, + "sectional": 27197, + "aegean": 27198, + "##cous": 27199, + "296": 27200, + "nylon": 27201, + "ghanaian": 27202, + "##egorical": 27203, + "##weig": 27204, + "cushions": 27205, + "forbid": 27206, + "fusiliers": 27207, + "obstruction": 27208, + "somerville": 27209, + "##scia": 27210, + "dime": 27211, + "earrings": 27212, + "elliptical": 27213, + "leyte": 27214, + "oder": 27215, + "polymers": 27216, + "timmy": 27217, + "atm": 27218, + "midtown": 27219, + "piloted": 27220, + "settles": 27221, + "continual": 27222, + "externally": 27223, + "mayfield": 27224, + "##uh": 27225, + "enrichment": 27226, + "henson": 27227, + "keane": 27228, + "persians": 27229, + "1733": 27230, + "benji": 27231, + "braden": 27232, + "pep": 27233, + "324": 27234, + "##efe": 27235, + "contenders": 27236, + "pepsi": 27237, + "valet": 27238, + "##isches": 27239, + "298": 27240, + "##asse": 27241, + "##earing": 27242, + "goofy": 27243, + "stroll": 27244, + "##amen": 27245, + "authoritarian": 27246, + "occurrences": 27247, + "adversary": 27248, + "ahmedabad": 27249, + "tangent": 27250, + "toppled": 27251, + "dorchester": 27252, + "1672": 27253, + "modernism": 27254, + "marxism": 27255, + "islamist": 27256, + "charlemagne": 27257, + "exponential": 27258, + "racks": 27259, + "unicode": 27260, + "brunette": 27261, + "mbc": 27262, + "pic": 27263, + "skirmish": 27264, + "##bund": 27265, + "##lad": 27266, + "##powered": 27267, + "##yst": 27268, + "hoisted": 27269, + "messina": 27270, + "shatter": 27271, + "##ctum": 27272, + "jedi": 27273, + "vantage": 27274, + "##music": 27275, + "##neil": 27276, + "clemens": 27277, + "mahmoud": 27278, + "corrupted": 27279, + "authentication": 27280, + "lowry": 27281, + "nils": 27282, + "##washed": 27283, + "omnibus": 27284, + "wounding": 27285, + "jillian": 27286, + "##itors": 27287, + "##opped": 27288, + "serialized": 27289, + "narcotics": 27290, + "handheld": 27291, + "##arm": 27292, + "##plicity": 27293, + "intersecting": 27294, + "stimulating": 27295, + "##onis": 27296, + "crate": 27297, + "fellowships": 27298, + "hemingway": 27299, + "casinos": 27300, + "climatic": 27301, + "fordham": 27302, + "copeland": 27303, + "drip": 27304, + "beatty": 27305, + "leaflets": 27306, + "robber": 27307, + "brothel": 27308, + "madeira": 27309, + "##hedral": 27310, + "sphinx": 27311, + "ultrasound": 27312, + "##vana": 27313, + "valor": 27314, + "forbade": 27315, + "leonid": 27316, + "villas": 27317, + "##aldo": 27318, + "duane": 27319, + "marquez": 27320, + "##cytes": 27321, + "disadvantaged": 27322, + "forearms": 27323, + "kawasaki": 27324, + "reacts": 27325, + "consular": 27326, + "lax": 27327, + "uncles": 27328, + "uphold": 27329, + "##hopper": 27330, + "concepcion": 27331, + "dorsey": 27332, + "lass": 27333, + "##izan": 27334, + "arching": 27335, + "passageway": 27336, + "1708": 27337, + "researches": 27338, + "tia": 27339, + "internationals": 27340, + "##graphs": 27341, + "##opers": 27342, + "distinguishes": 27343, + "javanese": 27344, + "divert": 27345, + "##uven": 27346, + "plotted": 27347, + "##listic": 27348, + "##rwin": 27349, + "##erik": 27350, + "##tify": 27351, + "affirmative": 27352, + "signifies": 27353, + "validation": 27354, + "##bson": 27355, + "kari": 27356, + "felicity": 27357, + "georgina": 27358, + "zulu": 27359, + "##eros": 27360, + "##rained": 27361, + "##rath": 27362, + "overcoming": 27363, + "##dot": 27364, + "argyll": 27365, + "##rbin": 27366, + "1734": 27367, + "chiba": 27368, + "ratification": 27369, + "windy": 27370, + "earls": 27371, + "parapet": 27372, + "##marks": 27373, + "hunan": 27374, + "pristine": 27375, + "astrid": 27376, + "punta": 27377, + "##gart": 27378, + "brodie": 27379, + "##kota": 27380, + "##oder": 27381, + "malaga": 27382, + "minerva": 27383, + "rouse": 27384, + "##phonic": 27385, + "bellowed": 27386, + "pagoda": 27387, + "portals": 27388, + "reclamation": 27389, + "##gur": 27390, + "##odies": 27391, + "##⁄₄": 27392, + "parentheses": 27393, + "quoting": 27394, + "allergic": 27395, + "palette": 27396, + "showcases": 27397, + "benefactor": 27398, + "heartland": 27399, + "nonlinear": 27400, + "##tness": 27401, + "bladed": 27402, + "cheerfully": 27403, + "scans": 27404, + "##ety": 27405, + "##hone": 27406, + "1666": 27407, + "girlfriends": 27408, + "pedersen": 27409, + "hiram": 27410, + "sous": 27411, + "##liche": 27412, + "##nator": 27413, + "1683": 27414, + "##nery": 27415, + "##orio": 27416, + "##umen": 27417, + "bobo": 27418, + "primaries": 27419, + "smiley": 27420, + "##cb": 27421, + "unearthed": 27422, + "uniformly": 27423, + "fis": 27424, + "metadata": 27425, + "1635": 27426, + "ind": 27427, + "##oted": 27428, + "recoil": 27429, + "##titles": 27430, + "##tura": 27431, + "##ια": 27432, + "406": 27433, + "hilbert": 27434, + "jamestown": 27435, + "mcmillan": 27436, + "tulane": 27437, + "seychelles": 27438, + "##frid": 27439, + "antics": 27440, + "coli": 27441, + "fated": 27442, + "stucco": 27443, + "##grants": 27444, + "1654": 27445, + "bulky": 27446, + "accolades": 27447, + "arrays": 27448, + "caledonian": 27449, + "carnage": 27450, + "optimism": 27451, + "puebla": 27452, + "##tative": 27453, + "##cave": 27454, + "enforcing": 27455, + "rotherham": 27456, + "seo": 27457, + "dunlop": 27458, + "aeronautics": 27459, + "chimed": 27460, + "incline": 27461, + "zoning": 27462, + "archduke": 27463, + "hellenistic": 27464, + "##oses": 27465, + "##sions": 27466, + "candi": 27467, + "thong": 27468, + "##ople": 27469, + "magnate": 27470, + "rustic": 27471, + "##rsk": 27472, + "projective": 27473, + "slant": 27474, + "##offs": 27475, + "danes": 27476, + "hollis": 27477, + "vocalists": 27478, + "##ammed": 27479, + "congenital": 27480, + "contend": 27481, + "gesellschaft": 27482, + "##ocating": 27483, + "##pressive": 27484, + "douglass": 27485, + "quieter": 27486, + "##cm": 27487, + "##kshi": 27488, + "howled": 27489, + "salim": 27490, + "spontaneously": 27491, + "townsville": 27492, + "buena": 27493, + "southport": 27494, + "##bold": 27495, + "kato": 27496, + "1638": 27497, + "faerie": 27498, + "stiffly": 27499, + "##vus": 27500, + "##rled": 27501, + "297": 27502, + "flawless": 27503, + "realising": 27504, + "taboo": 27505, + "##7th": 27506, + "bytes": 27507, + "straightening": 27508, + "356": 27509, + "jena": 27510, + "##hid": 27511, + "##rmin": 27512, + "cartwright": 27513, + "berber": 27514, + "bertram": 27515, + "soloists": 27516, + "411": 27517, + "noses": 27518, + "417": 27519, + "coping": 27520, + "fission": 27521, + "hardin": 27522, + "inca": 27523, + "##cen": 27524, + "1717": 27525, + "mobilized": 27526, + "vhf": 27527, + "##raf": 27528, + "biscuits": 27529, + "curate": 27530, + "##85": 27531, + "##anial": 27532, + "331": 27533, + "gaunt": 27534, + "neighbourhoods": 27535, + "1540": 27536, + "##abas": 27537, + "blanca": 27538, + "bypassed": 27539, + "sockets": 27540, + "behold": 27541, + "coincidentally": 27542, + "##bane": 27543, + "nara": 27544, + "shave": 27545, + "splinter": 27546, + "terrific": 27547, + "##arion": 27548, + "##erian": 27549, + "commonplace": 27550, + "juris": 27551, + "redwood": 27552, + "waistband": 27553, + "boxed": 27554, + "caitlin": 27555, + "fingerprints": 27556, + "jennie": 27557, + "naturalized": 27558, + "##ired": 27559, + "balfour": 27560, + "craters": 27561, + "jody": 27562, + "bungalow": 27563, + "hugely": 27564, + "quilt": 27565, + "glitter": 27566, + "pigeons": 27567, + "undertaker": 27568, + "bulging": 27569, + "constrained": 27570, + "goo": 27571, + "##sil": 27572, + "##akh": 27573, + "assimilation": 27574, + "reworked": 27575, + "##person": 27576, + "persuasion": 27577, + "##pants": 27578, + "felicia": 27579, + "##cliff": 27580, + "##ulent": 27581, + "1732": 27582, + "explodes": 27583, + "##dun": 27584, + "##inium": 27585, + "##zic": 27586, + "lyman": 27587, + "vulture": 27588, + "hog": 27589, + "overlook": 27590, + "begs": 27591, + "northwards": 27592, + "ow": 27593, + "spoil": 27594, + "##urer": 27595, + "fatima": 27596, + "favorably": 27597, + "accumulate": 27598, + "sargent": 27599, + "sorority": 27600, + "corresponded": 27601, + "dispersal": 27602, + "kochi": 27603, + "toned": 27604, + "##imi": 27605, + "##lita": 27606, + "internacional": 27607, + "newfound": 27608, + "##agger": 27609, + "##lynn": 27610, + "##rigue": 27611, + "booths": 27612, + "peanuts": 27613, + "##eborg": 27614, + "medicare": 27615, + "muriel": 27616, + "nur": 27617, + "##uram": 27618, + "crates": 27619, + "millennia": 27620, + "pajamas": 27621, + "worsened": 27622, + "##breakers": 27623, + "jimi": 27624, + "vanuatu": 27625, + "yawned": 27626, + "##udeau": 27627, + "carousel": 27628, + "##hony": 27629, + "hurdle": 27630, + "##ccus": 27631, + "##mounted": 27632, + "##pod": 27633, + "rv": 27634, + "##eche": 27635, + "airship": 27636, + "ambiguity": 27637, + "compulsion": 27638, + "recapture": 27639, + "##claiming": 27640, + "arthritis": 27641, + "##osomal": 27642, + "1667": 27643, + "asserting": 27644, + "ngc": 27645, + "sniffing": 27646, + "dade": 27647, + "discontent": 27648, + "glendale": 27649, + "ported": 27650, + "##amina": 27651, + "defamation": 27652, + "rammed": 27653, + "##scent": 27654, + "fling": 27655, + "livingstone": 27656, + "##fleet": 27657, + "875": 27658, + "##ppy": 27659, + "apocalyptic": 27660, + "comrade": 27661, + "lcd": 27662, + "##lowe": 27663, + "cessna": 27664, + "eine": 27665, + "persecuted": 27666, + "subsistence": 27667, + "demi": 27668, + "hoop": 27669, + "reliefs": 27670, + "710": 27671, + "coptic": 27672, + "progressing": 27673, + "stemmed": 27674, + "perpetrators": 27675, + "1665": 27676, + "priestess": 27677, + "##nio": 27678, + "dobson": 27679, + "ebony": 27680, + "rooster": 27681, + "itf": 27682, + "tortricidae": 27683, + "##bbon": 27684, + "##jian": 27685, + "cleanup": 27686, + "##jean": 27687, + "##øy": 27688, + "1721": 27689, + "eighties": 27690, + "taxonomic": 27691, + "holiness": 27692, + "##hearted": 27693, + "##spar": 27694, + "antilles": 27695, + "showcasing": 27696, + "stabilized": 27697, + "##nb": 27698, + "gia": 27699, + "mascara": 27700, + "michelangelo": 27701, + "dawned": 27702, + "##uria": 27703, + "##vinsky": 27704, + "extinguished": 27705, + "fitz": 27706, + "grotesque": 27707, + "£100": 27708, + "##fera": 27709, + "##loid": 27710, + "##mous": 27711, + "barges": 27712, + "neue": 27713, + "throbbed": 27714, + "cipher": 27715, + "johnnie": 27716, + "##a1": 27717, + "##mpt": 27718, + "outburst": 27719, + "##swick": 27720, + "spearheaded": 27721, + "administrations": 27722, + "c1": 27723, + "heartbreak": 27724, + "pixels": 27725, + "pleasantly": 27726, + "##enay": 27727, + "lombardy": 27728, + "plush": 27729, + "##nsed": 27730, + "bobbie": 27731, + "##hly": 27732, + "reapers": 27733, + "tremor": 27734, + "xiang": 27735, + "minogue": 27736, + "substantive": 27737, + "hitch": 27738, + "barak": 27739, + "##wyl": 27740, + "kwan": 27741, + "##encia": 27742, + "910": 27743, + "obscene": 27744, + "elegance": 27745, + "indus": 27746, + "surfer": 27747, + "bribery": 27748, + "conserve": 27749, + "##hyllum": 27750, + "##masters": 27751, + "horatio": 27752, + "##fat": 27753, + "apes": 27754, + "rebound": 27755, + "psychotic": 27756, + "##pour": 27757, + "iteration": 27758, + "##mium": 27759, + "##vani": 27760, + "botanic": 27761, + "horribly": 27762, + "antiques": 27763, + "dispose": 27764, + "paxton": 27765, + "##hli": 27766, + "##wg": 27767, + "timeless": 27768, + "1704": 27769, + "disregard": 27770, + "engraver": 27771, + "hounds": 27772, + "##bau": 27773, + "##version": 27774, + "looted": 27775, + "uno": 27776, + "facilitates": 27777, + "groans": 27778, + "masjid": 27779, + "rutland": 27780, + "antibody": 27781, + "disqualification": 27782, + "decatur": 27783, + "footballers": 27784, + "quake": 27785, + "slacks": 27786, + "48th": 27787, + "rein": 27788, + "scribe": 27789, + "stabilize": 27790, + "commits": 27791, + "exemplary": 27792, + "tho": 27793, + "##hort": 27794, + "##chison": 27795, + "pantry": 27796, + "traversed": 27797, + "##hiti": 27798, + "disrepair": 27799, + "identifiable": 27800, + "vibrated": 27801, + "baccalaureate": 27802, + "##nnis": 27803, + "csa": 27804, + "interviewing": 27805, + "##iensis": 27806, + "##raße": 27807, + "greaves": 27808, + "wealthiest": 27809, + "343": 27810, + "classed": 27811, + "jogged": 27812, + "£5": 27813, + "##58": 27814, + "##atal": 27815, + "illuminating": 27816, + "knicks": 27817, + "respecting": 27818, + "##uno": 27819, + "scrubbed": 27820, + "##iji": 27821, + "##dles": 27822, + "kruger": 27823, + "moods": 27824, + "growls": 27825, + "raider": 27826, + "silvia": 27827, + "chefs": 27828, + "kam": 27829, + "vr": 27830, + "cree": 27831, + "percival": 27832, + "##terol": 27833, + "gunter": 27834, + "counterattack": 27835, + "defiant": 27836, + "henan": 27837, + "ze": 27838, + "##rasia": 27839, + "##riety": 27840, + "equivalence": 27841, + "submissions": 27842, + "##fra": 27843, + "##thor": 27844, + "bautista": 27845, + "mechanically": 27846, + "##heater": 27847, + "cornice": 27848, + "herbal": 27849, + "templar": 27850, + "##mering": 27851, + "outputs": 27852, + "ruining": 27853, + "ligand": 27854, + "renumbered": 27855, + "extravagant": 27856, + "mika": 27857, + "blockbuster": 27858, + "eta": 27859, + "insurrection": 27860, + "##ilia": 27861, + "darkening": 27862, + "ferocious": 27863, + "pianos": 27864, + "strife": 27865, + "kinship": 27866, + "##aer": 27867, + "melee": 27868, + "##anor": 27869, + "##iste": 27870, + "##may": 27871, + "##oue": 27872, + "decidedly": 27873, + "weep": 27874, + "##jad": 27875, + "##missive": 27876, + "##ppel": 27877, + "354": 27878, + "puget": 27879, + "unease": 27880, + "##gnant": 27881, + "1629": 27882, + "hammering": 27883, + "kassel": 27884, + "ob": 27885, + "wessex": 27886, + "##lga": 27887, + "bromwich": 27888, + "egan": 27889, + "paranoia": 27890, + "utilization": 27891, + "##atable": 27892, + "##idad": 27893, + "contradictory": 27894, + "provoke": 27895, + "##ols": 27896, + "##ouring": 27897, + "##tangled": 27898, + "knesset": 27899, + "##very": 27900, + "##lette": 27901, + "plumbing": 27902, + "##sden": 27903, + "##¹": 27904, + "greensboro": 27905, + "occult": 27906, + "sniff": 27907, + "338": 27908, + "zev": 27909, + "beaming": 27910, + "gamer": 27911, + "haggard": 27912, + "mahal": 27913, + "##olt": 27914, + "##pins": 27915, + "mendes": 27916, + "utmost": 27917, + "briefing": 27918, + "gunnery": 27919, + "##gut": 27920, + "##pher": 27921, + "##zh": 27922, + "##rok": 27923, + "1679": 27924, + "khalifa": 27925, + "sonya": 27926, + "##boot": 27927, + "principals": 27928, + "urbana": 27929, + "wiring": 27930, + "##liffe": 27931, + "##minating": 27932, + "##rrado": 27933, + "dahl": 27934, + "nyu": 27935, + "skepticism": 27936, + "np": 27937, + "townspeople": 27938, + "ithaca": 27939, + "lobster": 27940, + "somethin": 27941, + "##fur": 27942, + "##arina": 27943, + "##−1": 27944, + "freighter": 27945, + "zimmerman": 27946, + "biceps": 27947, + "contractual": 27948, + "##herton": 27949, + "amend": 27950, + "hurrying": 27951, + "subconscious": 27952, + "##anal": 27953, + "336": 27954, + "meng": 27955, + "clermont": 27956, + "spawning": 27957, + "##eia": 27958, + "##lub": 27959, + "dignitaries": 27960, + "impetus": 27961, + "snacks": 27962, + "spotting": 27963, + "twigs": 27964, + "##bilis": 27965, + "##cz": 27966, + "##ouk": 27967, + "libertadores": 27968, + "nic": 27969, + "skylar": 27970, + "##aina": 27971, + "##firm": 27972, + "gustave": 27973, + "asean": 27974, + "##anum": 27975, + "dieter": 27976, + "legislatures": 27977, + "flirt": 27978, + "bromley": 27979, + "trolls": 27980, + "umar": 27981, + "##bbies": 27982, + "##tyle": 27983, + "blah": 27984, + "parc": 27985, + "bridgeport": 27986, + "crank": 27987, + "negligence": 27988, + "##nction": 27989, + "46th": 27990, + "constantin": 27991, + "molded": 27992, + "bandages": 27993, + "seriousness": 27994, + "00pm": 27995, + "siegel": 27996, + "carpets": 27997, + "compartments": 27998, + "upbeat": 27999, + "statehood": 28000, + "##dner": 28001, + "##edging": 28002, + "marko": 28003, + "730": 28004, + "platt": 28005, + "##hane": 28006, + "paving": 28007, + "##iy": 28008, + "1738": 28009, + "abbess": 28010, + "impatience": 28011, + "limousine": 28012, + "nbl": 28013, + "##talk": 28014, + "441": 28015, + "lucille": 28016, + "mojo": 28017, + "nightfall": 28018, + "robbers": 28019, + "##nais": 28020, + "karel": 28021, + "brisk": 28022, + "calves": 28023, + "replicate": 28024, + "ascribed": 28025, + "telescopes": 28026, + "##olf": 28027, + "intimidated": 28028, + "##reen": 28029, + "ballast": 28030, + "specialization": 28031, + "##sit": 28032, + "aerodynamic": 28033, + "caliphate": 28034, + "rainer": 28035, + "visionary": 28036, + "##arded": 28037, + "epsilon": 28038, + "##aday": 28039, + "##onte": 28040, + "aggregation": 28041, + "auditory": 28042, + "boosted": 28043, + "reunification": 28044, + "kathmandu": 28045, + "loco": 28046, + "robyn": 28047, + "402": 28048, + "acknowledges": 28049, + "appointing": 28050, + "humanoid": 28051, + "newell": 28052, + "redeveloped": 28053, + "restraints": 28054, + "##tained": 28055, + "barbarians": 28056, + "chopper": 28057, + "1609": 28058, + "italiana": 28059, + "##lez": 28060, + "##lho": 28061, + "investigates": 28062, + "wrestlemania": 28063, + "##anies": 28064, + "##bib": 28065, + "690": 28066, + "##falls": 28067, + "creaked": 28068, + "dragoons": 28069, + "gravely": 28070, + "minions": 28071, + "stupidity": 28072, + "volley": 28073, + "##harat": 28074, + "##week": 28075, + "musik": 28076, + "##eries": 28077, + "##uously": 28078, + "fungal": 28079, + "massimo": 28080, + "semantics": 28081, + "malvern": 28082, + "##ahl": 28083, + "##pee": 28084, + "discourage": 28085, + "embryo": 28086, + "imperialism": 28087, + "1910s": 28088, + "profoundly": 28089, + "##ddled": 28090, + "jiangsu": 28091, + "sparkled": 28092, + "stat": 28093, + "##holz": 28094, + "sweatshirt": 28095, + "tobin": 28096, + "##iction": 28097, + "sneered": 28098, + "##cheon": 28099, + "##oit": 28100, + "brit": 28101, + "causal": 28102, + "smyth": 28103, + "##neuve": 28104, + "diffuse": 28105, + "perrin": 28106, + "silvio": 28107, + "##ipes": 28108, + "##recht": 28109, + "detonated": 28110, + "iqbal": 28111, + "selma": 28112, + "##nism": 28113, + "##zumi": 28114, + "roasted": 28115, + "##riders": 28116, + "tay": 28117, + "##ados": 28118, + "##mament": 28119, + "##mut": 28120, + "##rud": 28121, + "840": 28122, + "completes": 28123, + "nipples": 28124, + "cfa": 28125, + "flavour": 28126, + "hirsch": 28127, + "##laus": 28128, + "calderon": 28129, + "sneakers": 28130, + "moravian": 28131, + "##ksha": 28132, + "1622": 28133, + "rq": 28134, + "294": 28135, + "##imeters": 28136, + "bodo": 28137, + "##isance": 28138, + "##pre": 28139, + "##ronia": 28140, + "anatomical": 28141, + "excerpt": 28142, + "##lke": 28143, + "dh": 28144, + "kunst": 28145, + "##tablished": 28146, + "##scoe": 28147, + "biomass": 28148, + "panted": 28149, + "unharmed": 28150, + "gael": 28151, + "housemates": 28152, + "montpellier": 28153, + "##59": 28154, + "coa": 28155, + "rodents": 28156, + "tonic": 28157, + "hickory": 28158, + "singleton": 28159, + "##taro": 28160, + "451": 28161, + "1719": 28162, + "aldo": 28163, + "breaststroke": 28164, + "dempsey": 28165, + "och": 28166, + "rocco": 28167, + "##cuit": 28168, + "merton": 28169, + "dissemination": 28170, + "midsummer": 28171, + "serials": 28172, + "##idi": 28173, + "haji": 28174, + "polynomials": 28175, + "##rdon": 28176, + "gs": 28177, + "enoch": 28178, + "prematurely": 28179, + "shutter": 28180, + "taunton": 28181, + "£3": 28182, + "##grating": 28183, + "##inates": 28184, + "archangel": 28185, + "harassed": 28186, + "##asco": 28187, + "326": 28188, + "archway": 28189, + "dazzling": 28190, + "##ecin": 28191, + "1736": 28192, + "sumo": 28193, + "wat": 28194, + "##kovich": 28195, + "1086": 28196, + "honneur": 28197, + "##ently": 28198, + "##nostic": 28199, + "##ttal": 28200, + "##idon": 28201, + "1605": 28202, + "403": 28203, + "1716": 28204, + "blogger": 28205, + "rents": 28206, + "##gnan": 28207, + "hires": 28208, + "##ikh": 28209, + "##dant": 28210, + "howie": 28211, + "##rons": 28212, + "handler": 28213, + "retracted": 28214, + "shocks": 28215, + "1632": 28216, + "arun": 28217, + "duluth": 28218, + "kepler": 28219, + "trumpeter": 28220, + "##lary": 28221, + "peeking": 28222, + "seasoned": 28223, + "trooper": 28224, + "##mara": 28225, + "laszlo": 28226, + "##iciencies": 28227, + "##rti": 28228, + "heterosexual": 28229, + "##inatory": 28230, + "##ssion": 28231, + "indira": 28232, + "jogging": 28233, + "##inga": 28234, + "##lism": 28235, + "beit": 28236, + "dissatisfaction": 28237, + "malice": 28238, + "##ately": 28239, + "nedra": 28240, + "peeling": 28241, + "##rgeon": 28242, + "47th": 28243, + "stadiums": 28244, + "475": 28245, + "vertigo": 28246, + "##ains": 28247, + "iced": 28248, + "restroom": 28249, + "##plify": 28250, + "##tub": 28251, + "illustrating": 28252, + "pear": 28253, + "##chner": 28254, + "##sibility": 28255, + "inorganic": 28256, + "rappers": 28257, + "receipts": 28258, + "watery": 28259, + "##kura": 28260, + "lucinda": 28261, + "##oulos": 28262, + "reintroduced": 28263, + "##8th": 28264, + "##tched": 28265, + "gracefully": 28266, + "saxons": 28267, + "nutritional": 28268, + "wastewater": 28269, + "rained": 28270, + "favourites": 28271, + "bedrock": 28272, + "fisted": 28273, + "hallways": 28274, + "likeness": 28275, + "upscale": 28276, + "##lateral": 28277, + "1580": 28278, + "blinds": 28279, + "prequel": 28280, + "##pps": 28281, + "##tama": 28282, + "deter": 28283, + "humiliating": 28284, + "restraining": 28285, + "tn": 28286, + "vents": 28287, + "1659": 28288, + "laundering": 28289, + "recess": 28290, + "rosary": 28291, + "tractors": 28292, + "coulter": 28293, + "federer": 28294, + "##ifiers": 28295, + "##plin": 28296, + "persistence": 28297, + "##quitable": 28298, + "geschichte": 28299, + "pendulum": 28300, + "quakers": 28301, + "##beam": 28302, + "bassett": 28303, + "pictorial": 28304, + "buffet": 28305, + "koln": 28306, + "##sitor": 28307, + "drills": 28308, + "reciprocal": 28309, + "shooters": 28310, + "##57": 28311, + "##cton": 28312, + "##tees": 28313, + "converge": 28314, + "pip": 28315, + "dmitri": 28316, + "donnelly": 28317, + "yamamoto": 28318, + "aqua": 28319, + "azores": 28320, + "demographics": 28321, + "hypnotic": 28322, + "spitfire": 28323, + "suspend": 28324, + "wryly": 28325, + "roderick": 28326, + "##rran": 28327, + "sebastien": 28328, + "##asurable": 28329, + "mavericks": 28330, + "##fles": 28331, + "##200": 28332, + "himalayan": 28333, + "prodigy": 28334, + "##iance": 28335, + "transvaal": 28336, + "demonstrators": 28337, + "handcuffs": 28338, + "dodged": 28339, + "mcnamara": 28340, + "sublime": 28341, + "1726": 28342, + "crazed": 28343, + "##efined": 28344, + "##till": 28345, + "ivo": 28346, + "pondered": 28347, + "reconciled": 28348, + "shrill": 28349, + "sava": 28350, + "##duk": 28351, + "bal": 28352, + "cad": 28353, + "heresy": 28354, + "jaipur": 28355, + "goran": 28356, + "##nished": 28357, + "341": 28358, + "lux": 28359, + "shelly": 28360, + "whitehall": 28361, + "##hre": 28362, + "israelis": 28363, + "peacekeeping": 28364, + "##wled": 28365, + "1703": 28366, + "demetrius": 28367, + "ousted": 28368, + "##arians": 28369, + "##zos": 28370, + "beale": 28371, + "anwar": 28372, + "backstroke": 28373, + "raged": 28374, + "shrinking": 28375, + "cremated": 28376, + "##yck": 28377, + "benign": 28378, + "towing": 28379, + "wadi": 28380, + "darmstadt": 28381, + "landfill": 28382, + "parana": 28383, + "soothe": 28384, + "colleen": 28385, + "sidewalks": 28386, + "mayfair": 28387, + "tumble": 28388, + "hepatitis": 28389, + "ferrer": 28390, + "superstructure": 28391, + "##gingly": 28392, + "##urse": 28393, + "##wee": 28394, + "anthropological": 28395, + "translators": 28396, + "##mies": 28397, + "closeness": 28398, + "hooves": 28399, + "##pw": 28400, + "mondays": 28401, + "##roll": 28402, + "##vita": 28403, + "landscaping": 28404, + "##urized": 28405, + "purification": 28406, + "sock": 28407, + "thorns": 28408, + "thwarted": 28409, + "jalan": 28410, + "tiberius": 28411, + "##taka": 28412, + "saline": 28413, + "##rito": 28414, + "confidently": 28415, + "khyber": 28416, + "sculptors": 28417, + "##ij": 28418, + "brahms": 28419, + "hammersmith": 28420, + "inspectors": 28421, + "battista": 28422, + "fivb": 28423, + "fragmentation": 28424, + "hackney": 28425, + "##uls": 28426, + "arresting": 28427, + "exercising": 28428, + "antoinette": 28429, + "bedfordshire": 28430, + "##zily": 28431, + "dyed": 28432, + "##hema": 28433, + "1656": 28434, + "racetrack": 28435, + "variability": 28436, + "##tique": 28437, + "1655": 28438, + "austrians": 28439, + "deteriorating": 28440, + "madman": 28441, + "theorists": 28442, + "aix": 28443, + "lehman": 28444, + "weathered": 28445, + "1731": 28446, + "decreed": 28447, + "eruptions": 28448, + "1729": 28449, + "flaw": 28450, + "quinlan": 28451, + "sorbonne": 28452, + "flutes": 28453, + "nunez": 28454, + "1711": 28455, + "adored": 28456, + "downwards": 28457, + "fable": 28458, + "rasped": 28459, + "1712": 28460, + "moritz": 28461, + "mouthful": 28462, + "renegade": 28463, + "shivers": 28464, + "stunts": 28465, + "dysfunction": 28466, + "restrain": 28467, + "translit": 28468, + "327": 28469, + "pancakes": 28470, + "##avio": 28471, + "##cision": 28472, + "##tray": 28473, + "351": 28474, + "vial": 28475, + "##lden": 28476, + "bain": 28477, + "##maid": 28478, + "##oxide": 28479, + "chihuahua": 28480, + "malacca": 28481, + "vimes": 28482, + "##rba": 28483, + "##rnier": 28484, + "1664": 28485, + "donnie": 28486, + "plaques": 28487, + "##ually": 28488, + "337": 28489, + "bangs": 28490, + "floppy": 28491, + "huntsville": 28492, + "loretta": 28493, + "nikolay": 28494, + "##otte": 28495, + "eater": 28496, + "handgun": 28497, + "ubiquitous": 28498, + "##hett": 28499, + "eras": 28500, + "zodiac": 28501, + "1634": 28502, + "##omorphic": 28503, + "1820s": 28504, + "##zog": 28505, + "cochran": 28506, + "##bula": 28507, + "##lithic": 28508, + "warring": 28509, + "##rada": 28510, + "dalai": 28511, + "excused": 28512, + "blazers": 28513, + "mcconnell": 28514, + "reeling": 28515, + "bot": 28516, + "este": 28517, + "##abi": 28518, + "geese": 28519, + "hoax": 28520, + "taxon": 28521, + "##bla": 28522, + "guitarists": 28523, + "##icon": 28524, + "condemning": 28525, + "hunts": 28526, + "inversion": 28527, + "moffat": 28528, + "taekwondo": 28529, + "##lvis": 28530, + "1624": 28531, + "stammered": 28532, + "##rest": 28533, + "##rzy": 28534, + "sousa": 28535, + "fundraiser": 28536, + "marylebone": 28537, + "navigable": 28538, + "uptown": 28539, + "cabbage": 28540, + "daniela": 28541, + "salman": 28542, + "shitty": 28543, + "whimper": 28544, + "##kian": 28545, + "##utive": 28546, + "programmers": 28547, + "protections": 28548, + "rm": 28549, + "##rmi": 28550, + "##rued": 28551, + "forceful": 28552, + "##enes": 28553, + "fuss": 28554, + "##tao": 28555, + "##wash": 28556, + "brat": 28557, + "oppressive": 28558, + "reykjavik": 28559, + "spartak": 28560, + "ticking": 28561, + "##inkles": 28562, + "##kiewicz": 28563, + "adolph": 28564, + "horst": 28565, + "maui": 28566, + "protege": 28567, + "straighten": 28568, + "cpc": 28569, + "landau": 28570, + "concourse": 28571, + "clements": 28572, + "resultant": 28573, + "##ando": 28574, + "imaginative": 28575, + "joo": 28576, + "reactivated": 28577, + "##rem": 28578, + "##ffled": 28579, + "##uising": 28580, + "consultative": 28581, + "##guide": 28582, + "flop": 28583, + "kaitlyn": 28584, + "mergers": 28585, + "parenting": 28586, + "somber": 28587, + "##vron": 28588, + "supervise": 28589, + "vidhan": 28590, + "##imum": 28591, + "courtship": 28592, + "exemplified": 28593, + "harmonies": 28594, + "medallist": 28595, + "refining": 28596, + "##rrow": 28597, + "##ка": 28598, + "amara": 28599, + "##hum": 28600, + "780": 28601, + "goalscorer": 28602, + "sited": 28603, + "overshadowed": 28604, + "rohan": 28605, + "displeasure": 28606, + "secretive": 28607, + "multiplied": 28608, + "osman": 28609, + "##orth": 28610, + "engravings": 28611, + "padre": 28612, + "##kali": 28613, + "##veda": 28614, + "miniatures": 28615, + "mis": 28616, + "##yala": 28617, + "clap": 28618, + "pali": 28619, + "rook": 28620, + "##cana": 28621, + "1692": 28622, + "57th": 28623, + "antennae": 28624, + "astro": 28625, + "oskar": 28626, + "1628": 28627, + "bulldog": 28628, + "crotch": 28629, + "hackett": 28630, + "yucatan": 28631, + "##sure": 28632, + "amplifiers": 28633, + "brno": 28634, + "ferrara": 28635, + "migrating": 28636, + "##gree": 28637, + "thanking": 28638, + "turing": 28639, + "##eza": 28640, + "mccann": 28641, + "ting": 28642, + "andersson": 28643, + "onslaught": 28644, + "gaines": 28645, + "ganga": 28646, + "incense": 28647, + "standardization": 28648, + "##mation": 28649, + "sentai": 28650, + "scuba": 28651, + "stuffing": 28652, + "turquoise": 28653, + "waivers": 28654, + "alloys": 28655, + "##vitt": 28656, + "regaining": 28657, + "vaults": 28658, + "##clops": 28659, + "##gizing": 28660, + "digger": 28661, + "furry": 28662, + "memorabilia": 28663, + "probing": 28664, + "##iad": 28665, + "payton": 28666, + "rec": 28667, + "deutschland": 28668, + "filippo": 28669, + "opaque": 28670, + "seamen": 28671, + "zenith": 28672, + "afrikaans": 28673, + "##filtration": 28674, + "disciplined": 28675, + "inspirational": 28676, + "##merie": 28677, + "banco": 28678, + "confuse": 28679, + "grafton": 28680, + "tod": 28681, + "##dgets": 28682, + "championed": 28683, + "simi": 28684, + "anomaly": 28685, + "biplane": 28686, + "##ceptive": 28687, + "electrode": 28688, + "##para": 28689, + "1697": 28690, + "cleavage": 28691, + "crossbow": 28692, + "swirl": 28693, + "informant": 28694, + "##lars": 28695, + "##osta": 28696, + "afi": 28697, + "bonfire": 28698, + "spec": 28699, + "##oux": 28700, + "lakeside": 28701, + "slump": 28702, + "##culus": 28703, + "##lais": 28704, + "##qvist": 28705, + "##rrigan": 28706, + "1016": 28707, + "facades": 28708, + "borg": 28709, + "inwardly": 28710, + "cervical": 28711, + "xl": 28712, + "pointedly": 28713, + "050": 28714, + "stabilization": 28715, + "##odon": 28716, + "chests": 28717, + "1699": 28718, + "hacked": 28719, + "ctv": 28720, + "orthogonal": 28721, + "suzy": 28722, + "##lastic": 28723, + "gaulle": 28724, + "jacobite": 28725, + "rearview": 28726, + "##cam": 28727, + "##erted": 28728, + "ashby": 28729, + "##drik": 28730, + "##igate": 28731, + "##mise": 28732, + "##zbek": 28733, + "affectionately": 28734, + "canine": 28735, + "disperse": 28736, + "latham": 28737, + "##istles": 28738, + "##ivar": 28739, + "spielberg": 28740, + "##orin": 28741, + "##idium": 28742, + "ezekiel": 28743, + "cid": 28744, + "##sg": 28745, + "durga": 28746, + "middletown": 28747, + "##cina": 28748, + "customized": 28749, + "frontiers": 28750, + "harden": 28751, + "##etano": 28752, + "##zzy": 28753, + "1604": 28754, + "bolsheviks": 28755, + "##66": 28756, + "coloration": 28757, + "yoko": 28758, + "##bedo": 28759, + "briefs": 28760, + "slabs": 28761, + "debra": 28762, + "liquidation": 28763, + "plumage": 28764, + "##oin": 28765, + "blossoms": 28766, + "dementia": 28767, + "subsidy": 28768, + "1611": 28769, + "proctor": 28770, + "relational": 28771, + "jerseys": 28772, + "parochial": 28773, + "ter": 28774, + "##ici": 28775, + "esa": 28776, + "peshawar": 28777, + "cavalier": 28778, + "loren": 28779, + "cpi": 28780, + "idiots": 28781, + "shamrock": 28782, + "1646": 28783, + "dutton": 28784, + "malabar": 28785, + "mustache": 28786, + "##endez": 28787, + "##ocytes": 28788, + "referencing": 28789, + "terminates": 28790, + "marche": 28791, + "yarmouth": 28792, + "##sop": 28793, + "acton": 28794, + "mated": 28795, + "seton": 28796, + "subtly": 28797, + "baptised": 28798, + "beige": 28799, + "extremes": 28800, + "jolted": 28801, + "kristina": 28802, + "telecast": 28803, + "##actic": 28804, + "safeguard": 28805, + "waldo": 28806, + "##baldi": 28807, + "##bular": 28808, + "endeavors": 28809, + "sloppy": 28810, + "subterranean": 28811, + "##ensburg": 28812, + "##itung": 28813, + "delicately": 28814, + "pigment": 28815, + "tq": 28816, + "##scu": 28817, + "1626": 28818, + "##ound": 28819, + "collisions": 28820, + "coveted": 28821, + "herds": 28822, + "##personal": 28823, + "##meister": 28824, + "##nberger": 28825, + "chopra": 28826, + "##ricting": 28827, + "abnormalities": 28828, + "defective": 28829, + "galician": 28830, + "lucie": 28831, + "##dilly": 28832, + "alligator": 28833, + "likened": 28834, + "##genase": 28835, + "burundi": 28836, + "clears": 28837, + "complexion": 28838, + "derelict": 28839, + "deafening": 28840, + "diablo": 28841, + "fingered": 28842, + "champaign": 28843, + "dogg": 28844, + "enlist": 28845, + "isotope": 28846, + "labeling": 28847, + "mrna": 28848, + "##erre": 28849, + "brilliance": 28850, + "marvelous": 28851, + "##ayo": 28852, + "1652": 28853, + "crawley": 28854, + "ether": 28855, + "footed": 28856, + "dwellers": 28857, + "deserts": 28858, + "hamish": 28859, + "rubs": 28860, + "warlock": 28861, + "skimmed": 28862, + "##lizer": 28863, + "870": 28864, + "buick": 28865, + "embark": 28866, + "heraldic": 28867, + "irregularities": 28868, + "##ajan": 28869, + "kiara": 28870, + "##kulam": 28871, + "##ieg": 28872, + "antigen": 28873, + "kowalski": 28874, + "##lge": 28875, + "oakley": 28876, + "visitation": 28877, + "##mbit": 28878, + "vt": 28879, + "##suit": 28880, + "1570": 28881, + "murderers": 28882, + "##miento": 28883, + "##rites": 28884, + "chimneys": 28885, + "##sling": 28886, + "condemn": 28887, + "custer": 28888, + "exchequer": 28889, + "havre": 28890, + "##ghi": 28891, + "fluctuations": 28892, + "##rations": 28893, + "dfb": 28894, + "hendricks": 28895, + "vaccines": 28896, + "##tarian": 28897, + "nietzsche": 28898, + "biking": 28899, + "juicy": 28900, + "##duced": 28901, + "brooding": 28902, + "scrolling": 28903, + "selangor": 28904, + "##ragan": 28905, + "352": 28906, + "annum": 28907, + "boomed": 28908, + "seminole": 28909, + "sugarcane": 28910, + "##dna": 28911, + "departmental": 28912, + "dismissing": 28913, + "innsbruck": 28914, + "arteries": 28915, + "ashok": 28916, + "batavia": 28917, + "daze": 28918, + "kun": 28919, + "overtook": 28920, + "##rga": 28921, + "##tlan": 28922, + "beheaded": 28923, + "gaddafi": 28924, + "holm": 28925, + "electronically": 28926, + "faulty": 28927, + "galilee": 28928, + "fractures": 28929, + "kobayashi": 28930, + "##lized": 28931, + "gunmen": 28932, + "magma": 28933, + "aramaic": 28934, + "mala": 28935, + "eastenders": 28936, + "inference": 28937, + "messengers": 28938, + "bf": 28939, + "##qu": 28940, + "407": 28941, + "bathrooms": 28942, + "##vere": 28943, + "1658": 28944, + "flashbacks": 28945, + "ideally": 28946, + "misunderstood": 28947, + "##jali": 28948, + "##weather": 28949, + "mendez": 28950, + "##grounds": 28951, + "505": 28952, + "uncanny": 28953, + "##iii": 28954, + "1709": 28955, + "friendships": 28956, + "##nbc": 28957, + "sacrament": 28958, + "accommodated": 28959, + "reiterated": 28960, + "logistical": 28961, + "pebbles": 28962, + "thumped": 28963, + "##escence": 28964, + "administering": 28965, + "decrees": 28966, + "drafts": 28967, + "##flight": 28968, + "##cased": 28969, + "##tula": 28970, + "futuristic": 28971, + "picket": 28972, + "intimidation": 28973, + "winthrop": 28974, + "##fahan": 28975, + "interfered": 28976, + "339": 28977, + "afar": 28978, + "francoise": 28979, + "morally": 28980, + "uta": 28981, + "cochin": 28982, + "croft": 28983, + "dwarfs": 28984, + "##bruck": 28985, + "##dents": 28986, + "##nami": 28987, + "biker": 28988, + "##hner": 28989, + "##meral": 28990, + "nano": 28991, + "##isen": 28992, + "##ometric": 28993, + "##pres": 28994, + "##ан": 28995, + "brightened": 28996, + "meek": 28997, + "parcels": 28998, + "securely": 28999, + "gunners": 29000, + "##jhl": 29001, + "##zko": 29002, + "agile": 29003, + "hysteria": 29004, + "##lten": 29005, + "##rcus": 29006, + "bukit": 29007, + "champs": 29008, + "chevy": 29009, + "cuckoo": 29010, + "leith": 29011, + "sadler": 29012, + "theologians": 29013, + "welded": 29014, + "##section": 29015, + "1663": 29016, + "jj": 29017, + "plurality": 29018, + "xander": 29019, + "##rooms": 29020, + "##formed": 29021, + "shredded": 29022, + "temps": 29023, + "intimately": 29024, + "pau": 29025, + "tormented": 29026, + "##lok": 29027, + "##stellar": 29028, + "1618": 29029, + "charred": 29030, + "ems": 29031, + "essen": 29032, + "##mmel": 29033, + "alarms": 29034, + "spraying": 29035, + "ascot": 29036, + "blooms": 29037, + "twinkle": 29038, + "##abia": 29039, + "##apes": 29040, + "internment": 29041, + "obsidian": 29042, + "##chaft": 29043, + "snoop": 29044, + "##dav": 29045, + "##ooping": 29046, + "malibu": 29047, + "##tension": 29048, + "quiver": 29049, + "##itia": 29050, + "hays": 29051, + "mcintosh": 29052, + "travers": 29053, + "walsall": 29054, + "##ffie": 29055, + "1623": 29056, + "beverley": 29057, + "schwarz": 29058, + "plunging": 29059, + "structurally": 29060, + "m3": 29061, + "rosenthal": 29062, + "vikram": 29063, + "##tsk": 29064, + "770": 29065, + "ghz": 29066, + "##onda": 29067, + "##tiv": 29068, + "chalmers": 29069, + "groningen": 29070, + "pew": 29071, + "reckon": 29072, + "unicef": 29073, + "##rvis": 29074, + "55th": 29075, + "##gni": 29076, + "1651": 29077, + "sulawesi": 29078, + "avila": 29079, + "cai": 29080, + "metaphysical": 29081, + "screwing": 29082, + "turbulence": 29083, + "##mberg": 29084, + "augusto": 29085, + "samba": 29086, + "56th": 29087, + "baffled": 29088, + "momentary": 29089, + "toxin": 29090, + "##urian": 29091, + "##wani": 29092, + "aachen": 29093, + "condoms": 29094, + "dali": 29095, + "steppe": 29096, + "##3d": 29097, + "##app": 29098, + "##oed": 29099, + "##year": 29100, + "adolescence": 29101, + "dauphin": 29102, + "electrically": 29103, + "inaccessible": 29104, + "microscopy": 29105, + "nikita": 29106, + "##ega": 29107, + "atv": 29108, + "##cel": 29109, + "##enter": 29110, + "##oles": 29111, + "##oteric": 29112, + "##ы": 29113, + "accountants": 29114, + "punishments": 29115, + "wrongly": 29116, + "bribes": 29117, + "adventurous": 29118, + "clinch": 29119, + "flinders": 29120, + "southland": 29121, + "##hem": 29122, + "##kata": 29123, + "gough": 29124, + "##ciency": 29125, + "lads": 29126, + "soared": 29127, + "##ה": 29128, + "undergoes": 29129, + "deformation": 29130, + "outlawed": 29131, + "rubbish": 29132, + "##arus": 29133, + "##mussen": 29134, + "##nidae": 29135, + "##rzburg": 29136, + "arcs": 29137, + "##ingdon": 29138, + "##tituted": 29139, + "1695": 29140, + "wheelbase": 29141, + "wheeling": 29142, + "bombardier": 29143, + "campground": 29144, + "zebra": 29145, + "##lices": 29146, + "##oj": 29147, + "##bain": 29148, + "lullaby": 29149, + "##ecure": 29150, + "donetsk": 29151, + "wylie": 29152, + "grenada": 29153, + "##arding": 29154, + "##ης": 29155, + "squinting": 29156, + "eireann": 29157, + "opposes": 29158, + "##andra": 29159, + "maximal": 29160, + "runes": 29161, + "##broken": 29162, + "##cuting": 29163, + "##iface": 29164, + "##ror": 29165, + "##rosis": 29166, + "additive": 29167, + "britney": 29168, + "adultery": 29169, + "triggering": 29170, + "##drome": 29171, + "detrimental": 29172, + "aarhus": 29173, + "containment": 29174, + "jc": 29175, + "swapped": 29176, + "vichy": 29177, + "##ioms": 29178, + "madly": 29179, + "##oric": 29180, + "##rag": 29181, + "brant": 29182, + "##ckey": 29183, + "##trix": 29184, + "1560": 29185, + "1612": 29186, + "broughton": 29187, + "rustling": 29188, + "##stems": 29189, + "##uder": 29190, + "asbestos": 29191, + "mentoring": 29192, + "##nivorous": 29193, + "finley": 29194, + "leaps": 29195, + "##isan": 29196, + "apical": 29197, + "pry": 29198, + "slits": 29199, + "substitutes": 29200, + "##dict": 29201, + "intuitive": 29202, + "fantasia": 29203, + "insistent": 29204, + "unreasonable": 29205, + "##igen": 29206, + "##vna": 29207, + "domed": 29208, + "hannover": 29209, + "margot": 29210, + "ponder": 29211, + "##zziness": 29212, + "impromptu": 29213, + "jian": 29214, + "lc": 29215, + "rampage": 29216, + "stemming": 29217, + "##eft": 29218, + "andrey": 29219, + "gerais": 29220, + "whichever": 29221, + "amnesia": 29222, + "appropriated": 29223, + "anzac": 29224, + "clicks": 29225, + "modifying": 29226, + "ultimatum": 29227, + "cambrian": 29228, + "maids": 29229, + "verve": 29230, + "yellowstone": 29231, + "##mbs": 29232, + "conservatoire": 29233, + "##scribe": 29234, + "adherence": 29235, + "dinners": 29236, + "spectra": 29237, + "imperfect": 29238, + "mysteriously": 29239, + "sidekick": 29240, + "tatar": 29241, + "tuba": 29242, + "##aks": 29243, + "##ifolia": 29244, + "distrust": 29245, + "##athan": 29246, + "##zle": 29247, + "c2": 29248, + "ronin": 29249, + "zac": 29250, + "##pse": 29251, + "celaena": 29252, + "instrumentalist": 29253, + "scents": 29254, + "skopje": 29255, + "##mbling": 29256, + "comical": 29257, + "compensated": 29258, + "vidal": 29259, + "condor": 29260, + "intersect": 29261, + "jingle": 29262, + "wavelengths": 29263, + "##urrent": 29264, + "mcqueen": 29265, + "##izzly": 29266, + "carp": 29267, + "weasel": 29268, + "422": 29269, + "kanye": 29270, + "militias": 29271, + "postdoctoral": 29272, + "eugen": 29273, + "gunslinger": 29274, + "##ɛ": 29275, + "faux": 29276, + "hospice": 29277, + "##for": 29278, + "appalled": 29279, + "derivation": 29280, + "dwarves": 29281, + "##elis": 29282, + "dilapidated": 29283, + "##folk": 29284, + "astoria": 29285, + "philology": 29286, + "##lwyn": 29287, + "##otho": 29288, + "##saka": 29289, + "inducing": 29290, + "philanthropy": 29291, + "##bf": 29292, + "##itative": 29293, + "geek": 29294, + "markedly": 29295, + "sql": 29296, + "##yce": 29297, + "bessie": 29298, + "indices": 29299, + "rn": 29300, + "##flict": 29301, + "495": 29302, + "frowns": 29303, + "resolving": 29304, + "weightlifting": 29305, + "tugs": 29306, + "cleric": 29307, + "contentious": 29308, + "1653": 29309, + "mania": 29310, + "rms": 29311, + "##miya": 29312, + "##reate": 29313, + "##ruck": 29314, + "##tucket": 29315, + "bien": 29316, + "eels": 29317, + "marek": 29318, + "##ayton": 29319, + "##cence": 29320, + "discreet": 29321, + "unofficially": 29322, + "##ife": 29323, + "leaks": 29324, + "##bber": 29325, + "1705": 29326, + "332": 29327, + "dung": 29328, + "compressor": 29329, + "hillsborough": 29330, + "pandit": 29331, + "shillings": 29332, + "distal": 29333, + "##skin": 29334, + "381": 29335, + "##tat": 29336, + "##you": 29337, + "nosed": 29338, + "##nir": 29339, + "mangrove": 29340, + "undeveloped": 29341, + "##idia": 29342, + "textures": 29343, + "##inho": 29344, + "##500": 29345, + "##rise": 29346, + "ae": 29347, + "irritating": 29348, + "nay": 29349, + "amazingly": 29350, + "bancroft": 29351, + "apologetic": 29352, + "compassionate": 29353, + "kata": 29354, + "symphonies": 29355, + "##lovic": 29356, + "airspace": 29357, + "##lch": 29358, + "930": 29359, + "gifford": 29360, + "precautions": 29361, + "fulfillment": 29362, + "sevilla": 29363, + "vulgar": 29364, + "martinique": 29365, + "##urities": 29366, + "looting": 29367, + "piccolo": 29368, + "tidy": 29369, + "##dermott": 29370, + "quadrant": 29371, + "armchair": 29372, + "incomes": 29373, + "mathematicians": 29374, + "stampede": 29375, + "nilsson": 29376, + "##inking": 29377, + "##scan": 29378, + "foo": 29379, + "quarterfinal": 29380, + "##ostal": 29381, + "shang": 29382, + "shouldered": 29383, + "squirrels": 29384, + "##owe": 29385, + "344": 29386, + "vinegar": 29387, + "##bner": 29388, + "##rchy": 29389, + "##systems": 29390, + "delaying": 29391, + "##trics": 29392, + "ars": 29393, + "dwyer": 29394, + "rhapsody": 29395, + "sponsoring": 29396, + "##gration": 29397, + "bipolar": 29398, + "cinder": 29399, + "starters": 29400, + "##olio": 29401, + "##urst": 29402, + "421": 29403, + "signage": 29404, + "##nty": 29405, + "aground": 29406, + "figurative": 29407, + "mons": 29408, + "acquaintances": 29409, + "duets": 29410, + "erroneously": 29411, + "soyuz": 29412, + "elliptic": 29413, + "recreated": 29414, + "##cultural": 29415, + "##quette": 29416, + "##ssed": 29417, + "##tma": 29418, + "##zcz": 29419, + "moderator": 29420, + "scares": 29421, + "##itaire": 29422, + "##stones": 29423, + "##udence": 29424, + "juniper": 29425, + "sighting": 29426, + "##just": 29427, + "##nsen": 29428, + "britten": 29429, + "calabria": 29430, + "ry": 29431, + "bop": 29432, + "cramer": 29433, + "forsyth": 29434, + "stillness": 29435, + "##л": 29436, + "airmen": 29437, + "gathers": 29438, + "unfit": 29439, + "##umber": 29440, + "##upt": 29441, + "taunting": 29442, + "##rip": 29443, + "seeker": 29444, + "streamlined": 29445, + "##bution": 29446, + "holster": 29447, + "schumann": 29448, + "tread": 29449, + "vox": 29450, + "##gano": 29451, + "##onzo": 29452, + "strive": 29453, + "dil": 29454, + "reforming": 29455, + "covent": 29456, + "newbury": 29457, + "predicting": 29458, + "##orro": 29459, + "decorate": 29460, + "tre": 29461, + "##puted": 29462, + "andover": 29463, + "ie": 29464, + "asahi": 29465, + "dept": 29466, + "dunkirk": 29467, + "gills": 29468, + "##tori": 29469, + "buren": 29470, + "huskies": 29471, + "##stis": 29472, + "##stov": 29473, + "abstracts": 29474, + "bets": 29475, + "loosen": 29476, + "##opa": 29477, + "1682": 29478, + "yearning": 29479, + "##glio": 29480, + "##sir": 29481, + "berman": 29482, + "effortlessly": 29483, + "enamel": 29484, + "napoli": 29485, + "persist": 29486, + "##peration": 29487, + "##uez": 29488, + "attache": 29489, + "elisa": 29490, + "b1": 29491, + "invitations": 29492, + "##kic": 29493, + "accelerating": 29494, + "reindeer": 29495, + "boardwalk": 29496, + "clutches": 29497, + "nelly": 29498, + "polka": 29499, + "starbucks": 29500, + "##kei": 29501, + "adamant": 29502, + "huey": 29503, + "lough": 29504, + "unbroken": 29505, + "adventurer": 29506, + "embroidery": 29507, + "inspecting": 29508, + "stanza": 29509, + "##ducted": 29510, + "naia": 29511, + "taluka": 29512, + "##pone": 29513, + "##roids": 29514, + "chases": 29515, + "deprivation": 29516, + "florian": 29517, + "##jing": 29518, + "##ppet": 29519, + "earthly": 29520, + "##lib": 29521, + "##ssee": 29522, + "colossal": 29523, + "foreigner": 29524, + "vet": 29525, + "freaks": 29526, + "patrice": 29527, + "rosewood": 29528, + "triassic": 29529, + "upstate": 29530, + "##pkins": 29531, + "dominates": 29532, + "ata": 29533, + "chants": 29534, + "ks": 29535, + "vo": 29536, + "##400": 29537, + "##bley": 29538, + "##raya": 29539, + "##rmed": 29540, + "555": 29541, + "agra": 29542, + "infiltrate": 29543, + "##ailing": 29544, + "##ilation": 29545, + "##tzer": 29546, + "##uppe": 29547, + "##werk": 29548, + "binoculars": 29549, + "enthusiast": 29550, + "fujian": 29551, + "squeak": 29552, + "##avs": 29553, + "abolitionist": 29554, + "almeida": 29555, + "boredom": 29556, + "hampstead": 29557, + "marsden": 29558, + "rations": 29559, + "##ands": 29560, + "inflated": 29561, + "334": 29562, + "bonuses": 29563, + "rosalie": 29564, + "patna": 29565, + "##rco": 29566, + "329": 29567, + "detachments": 29568, + "penitentiary": 29569, + "54th": 29570, + "flourishing": 29571, + "woolf": 29572, + "##dion": 29573, + "##etched": 29574, + "papyrus": 29575, + "##lster": 29576, + "##nsor": 29577, + "##toy": 29578, + "bobbed": 29579, + "dismounted": 29580, + "endelle": 29581, + "inhuman": 29582, + "motorola": 29583, + "tbs": 29584, + "wince": 29585, + "wreath": 29586, + "##ticus": 29587, + "hideout": 29588, + "inspections": 29589, + "sanjay": 29590, + "disgrace": 29591, + "infused": 29592, + "pudding": 29593, + "stalks": 29594, + "##urbed": 29595, + "arsenic": 29596, + "leases": 29597, + "##hyl": 29598, + "##rrard": 29599, + "collarbone": 29600, + "##waite": 29601, + "##wil": 29602, + "dowry": 29603, + "##bant": 29604, + "##edance": 29605, + "genealogical": 29606, + "nitrate": 29607, + "salamanca": 29608, + "scandals": 29609, + "thyroid": 29610, + "necessitated": 29611, + "##!": 29612, + "##\"": 29613, + "###": 29614, + "##$": 29615, + "##%": 29616, + "##&": 29617, + "##'": 29618, + "##(": 29619, + "##)": 29620, + "##*": 29621, + "##+": 29622, + "##,": 29623, + "##-": 29624, + "##.": 29625, + "##/": 29626, + "##:": 29627, + "##;": 29628, + "##<": 29629, + "##=": 29630, + "##>": 29631, + "##?": 29632, + "##@": 29633, + "##[": 29634, + "##\\": 29635, + "##]": 29636, + "##^": 29637, + "##_": 29638, + "##`": 29639, + "##{": 29640, + "##|": 29641, + "##}": 29642, + "##~": 29643, + "##¡": 29644, + "##¢": 29645, + "##£": 29646, + "##¤": 29647, + "##¥": 29648, + "##¦": 29649, + "##§": 29650, + "##¨": 29651, + "##©": 29652, + "##ª": 29653, + "##«": 29654, + "##¬": 29655, + "##®": 29656, + "##±": 29657, + "##´": 29658, + "##µ": 29659, + "##¶": 29660, + "##·": 29661, + "##º": 29662, + "##»": 29663, + "##¼": 29664, + "##¾": 29665, + "##¿": 29666, + "##æ": 29667, + "##ð": 29668, + "##÷": 29669, + "##þ": 29670, + "##đ": 29671, + "##ħ": 29672, + "##ŋ": 29673, + "##œ": 29674, + "##ƒ": 29675, + "##ɐ": 29676, + "##ɑ": 29677, + "##ɒ": 29678, + "##ɔ": 29679, + "##ɕ": 29680, + "##ə": 29681, + "##ɡ": 29682, + "##ɣ": 29683, + "##ɨ": 29684, + "##ɪ": 29685, + "##ɫ": 29686, + "##ɬ": 29687, + "##ɯ": 29688, + "##ɲ": 29689, + "##ɴ": 29690, + "##ɹ": 29691, + "##ɾ": 29692, + "##ʀ": 29693, + "##ʁ": 29694, + "##ʂ": 29695, + "##ʃ": 29696, + "##ʉ": 29697, + "##ʊ": 29698, + "##ʋ": 29699, + "##ʌ": 29700, + "##ʎ": 29701, + "##ʐ": 29702, + "##ʑ": 29703, + "##ʒ": 29704, + "##ʔ": 29705, + "##ʰ": 29706, + "##ʲ": 29707, + "##ʳ": 29708, + "##ʷ": 29709, + "##ʸ": 29710, + "##ʻ": 29711, + "##ʼ": 29712, + "##ʾ": 29713, + "##ʿ": 29714, + "##ˈ": 29715, + "##ˡ": 29716, + "##ˢ": 29717, + "##ˣ": 29718, + "##ˤ": 29719, + "##β": 29720, + "##γ": 29721, + "##δ": 29722, + "##ε": 29723, + "##ζ": 29724, + "##θ": 29725, + "##κ": 29726, + "##λ": 29727, + "##μ": 29728, + "##ξ": 29729, + "##ο": 29730, + "##π": 29731, + "##ρ": 29732, + "##σ": 29733, + "##τ": 29734, + "##υ": 29735, + "##φ": 29736, + "##χ": 29737, + "##ψ": 29738, + "##ω": 29739, + "##б": 29740, + "##г": 29741, + "##д": 29742, + "##ж": 29743, + "##з": 29744, + "##м": 29745, + "##п": 29746, + "##с": 29747, + "##у": 29748, + "##ф": 29749, + "##х": 29750, + "##ц": 29751, + "##ч": 29752, + "##ш": 29753, + "##щ": 29754, + "##ъ": 29755, + "##э": 29756, + "##ю": 29757, + "##ђ": 29758, + "##є": 29759, + "##і": 29760, + "##ј": 29761, + "##љ": 29762, + "##њ": 29763, + "##ћ": 29764, + "##ӏ": 29765, + "##ա": 29766, + "##բ": 29767, + "##գ": 29768, + "##դ": 29769, + "##ե": 29770, + "##թ": 29771, + "##ի": 29772, + "##լ": 29773, + "##կ": 29774, + "##հ": 29775, + "##մ": 29776, + "##յ": 29777, + "##ն": 29778, + "##ո": 29779, + "##պ": 29780, + "##ս": 29781, + "##վ": 29782, + "##տ": 29783, + "##ր": 29784, + "##ւ": 29785, + "##ք": 29786, + "##־": 29787, + "##א": 29788, + "##ב": 29789, + "##ג": 29790, + "##ד": 29791, + "##ו": 29792, + "##ז": 29793, + "##ח": 29794, + "##ט": 29795, + "##י": 29796, + "##ך": 29797, + "##כ": 29798, + "##ל": 29799, + "##ם": 29800, + "##מ": 29801, + "##ן": 29802, + "##נ": 29803, + "##ס": 29804, + "##ע": 29805, + "##ף": 29806, + "##פ": 29807, + "##ץ": 29808, + "##צ": 29809, + "##ק": 29810, + "##ר": 29811, + "##ש": 29812, + "##ת": 29813, + "##،": 29814, + "##ء": 29815, + "##ب": 29816, + "##ت": 29817, + "##ث": 29818, + "##ج": 29819, + "##ح": 29820, + "##خ": 29821, + "##ذ": 29822, + "##ز": 29823, + "##س": 29824, + "##ش": 29825, + "##ص": 29826, + "##ض": 29827, + "##ط": 29828, + "##ظ": 29829, + "##ع": 29830, + "##غ": 29831, + "##ـ": 29832, + "##ف": 29833, + "##ق": 29834, + "##ك": 29835, + "##و": 29836, + "##ى": 29837, + "##ٹ": 29838, + "##پ": 29839, + "##چ": 29840, + "##ک": 29841, + "##گ": 29842, + "##ں": 29843, + "##ھ": 29844, + "##ہ": 29845, + "##ے": 29846, + "##अ": 29847, + "##आ": 29848, + "##उ": 29849, + "##ए": 29850, + "##क": 29851, + "##ख": 29852, + "##ग": 29853, + "##च": 29854, + "##ज": 29855, + "##ट": 29856, + "##ड": 29857, + "##ण": 29858, + "##त": 29859, + "##थ": 29860, + "##द": 29861, + "##ध": 29862, + "##न": 29863, + "##प": 29864, + "##ब": 29865, + "##भ": 29866, + "##म": 29867, + "##य": 29868, + "##र": 29869, + "##ल": 29870, + "##व": 29871, + "##श": 29872, + "##ष": 29873, + "##स": 29874, + "##ह": 29875, + "##ा": 29876, + "##ि": 29877, + "##ी": 29878, + "##ो": 29879, + "##।": 29880, + "##॥": 29881, + "##ং": 29882, + "##অ": 29883, + "##আ": 29884, + "##ই": 29885, + "##উ": 29886, + "##এ": 29887, + "##ও": 29888, + "##ক": 29889, + "##খ": 29890, + "##গ": 29891, + "##চ": 29892, + "##ছ": 29893, + "##জ": 29894, + "##ট": 29895, + "##ড": 29896, + "##ণ": 29897, + "##ত": 29898, + "##থ": 29899, + "##দ": 29900, + "##ধ": 29901, + "##ন": 29902, + "##প": 29903, + "##ব": 29904, + "##ভ": 29905, + "##ম": 29906, + "##য": 29907, + "##র": 29908, + "##ল": 29909, + "##শ": 29910, + "##ষ": 29911, + "##স": 29912, + "##হ": 29913, + "##া": 29914, + "##ি": 29915, + "##ী": 29916, + "##ে": 29917, + "##க": 29918, + "##ச": 29919, + "##ட": 29920, + "##த": 29921, + "##ந": 29922, + "##ன": 29923, + "##ப": 29924, + "##ம": 29925, + "##ய": 29926, + "##ர": 29927, + "##ல": 29928, + "##ள": 29929, + "##வ": 29930, + "##ா": 29931, + "##ி": 29932, + "##ு": 29933, + "##ே": 29934, + "##ை": 29935, + "##ನ": 29936, + "##ರ": 29937, + "##ಾ": 29938, + "##ක": 29939, + "##ය": 29940, + "##ර": 29941, + "##ල": 29942, + "##ව": 29943, + "##ා": 29944, + "##ก": 29945, + "##ง": 29946, + "##ต": 29947, + "##ท": 29948, + "##น": 29949, + "##พ": 29950, + "##ม": 29951, + "##ย": 29952, + "##ร": 29953, + "##ล": 29954, + "##ว": 29955, + "##ส": 29956, + "##อ": 29957, + "##า": 29958, + "##เ": 29959, + "##་": 29960, + "##།": 29961, + "##ག": 29962, + "##ང": 29963, + "##ད": 29964, + "##ན": 29965, + "##པ": 29966, + "##བ": 29967, + "##མ": 29968, + "##འ": 29969, + "##ར": 29970, + "##ལ": 29971, + "##ས": 29972, + "##မ": 29973, + "##ა": 29974, + "##ბ": 29975, + "##გ": 29976, + "##დ": 29977, + "##ე": 29978, + "##ვ": 29979, + "##თ": 29980, + "##ი": 29981, + "##კ": 29982, + "##ლ": 29983, + "##მ": 29984, + "##ნ": 29985, + "##ო": 29986, + "##რ": 29987, + "##ს": 29988, + "##ტ": 29989, + "##უ": 29990, + "##ᄀ": 29991, + "##ᄂ": 29992, + "##ᄃ": 29993, + "##ᄅ": 29994, + "##ᄆ": 29995, + "##ᄇ": 29996, + "##ᄉ": 29997, + "##ᄊ": 29998, + "##ᄋ": 29999, + "##ᄌ": 30000, + "##ᄎ": 30001, + "##ᄏ": 30002, + "##ᄐ": 30003, + "##ᄑ": 30004, + "##ᄒ": 30005, + "##ᅡ": 30006, + "##ᅢ": 30007, + "##ᅥ": 30008, + "##ᅦ": 30009, + "##ᅧ": 30010, + "##ᅩ": 30011, + "##ᅪ": 30012, + "##ᅭ": 30013, + "##ᅮ": 30014, + "##ᅯ": 30015, + "##ᅲ": 30016, + "##ᅳ": 30017, + "##ᅴ": 30018, + "##ᅵ": 30019, + "##ᆨ": 30020, + "##ᆫ": 30021, + "##ᆯ": 30022, + "##ᆷ": 30023, + "##ᆸ": 30024, + "##ᆼ": 30025, + "##ᴬ": 30026, + "##ᴮ": 30027, + "##ᴰ": 30028, + "##ᴵ": 30029, + "##ᴺ": 30030, + "##ᵀ": 30031, + "##ᵃ": 30032, + "##ᵇ": 30033, + "##ᵈ": 30034, + "##ᵉ": 30035, + "##ᵍ": 30036, + "##ᵏ": 30037, + "##ᵐ": 30038, + "##ᵒ": 30039, + "##ᵖ": 30040, + "##ᵗ": 30041, + "##ᵘ": 30042, + "##ᵣ": 30043, + "##ᵤ": 30044, + "##ᵥ": 30045, + "##ᶜ": 30046, + "##ᶠ": 30047, + "##‐": 30048, + "##‑": 30049, + "##‒": 30050, + "##–": 30051, + "##—": 30052, + "##―": 30053, + "##‖": 30054, + "##‘": 30055, + "##’": 30056, + "##‚": 30057, + "##“": 30058, + "##”": 30059, + "##„": 30060, + "##†": 30061, + "##‡": 30062, + "##•": 30063, + "##…": 30064, + "##‰": 30065, + "##′": 30066, + "##″": 30067, + "##›": 30068, + "##‿": 30069, + "##⁄": 30070, + "##⁰": 30071, + "##ⁱ": 30072, + "##⁴": 30073, + "##⁵": 30074, + "##⁶": 30075, + "##⁷": 30076, + "##⁸": 30077, + "##⁹": 30078, + "##⁻": 30079, + "##ⁿ": 30080, + "##₅": 30081, + "##₆": 30082, + "##₇": 30083, + "##₈": 30084, + "##₉": 30085, + "##₊": 30086, + "##₍": 30087, + "##₎": 30088, + "##ₐ": 30089, + "##ₑ": 30090, + "##ₒ": 30091, + "##ₓ": 30092, + "##ₕ": 30093, + "##ₖ": 30094, + "##ₗ": 30095, + "##ₘ": 30096, + "##ₚ": 30097, + "##ₛ": 30098, + "##ₜ": 30099, + "##₤": 30100, + "##₩": 30101, + "##€": 30102, + "##₱": 30103, + "##₹": 30104, + "##ℓ": 30105, + "##№": 30106, + "##ℝ": 30107, + "##™": 30108, + "##⅓": 30109, + "##⅔": 30110, + "##←": 30111, + "##↑": 30112, + "##→": 30113, + "##↓": 30114, + "##↔": 30115, + "##↦": 30116, + "##⇄": 30117, + "##⇌": 30118, + "##⇒": 30119, + "##∂": 30120, + "##∅": 30121, + "##∆": 30122, + "##∇": 30123, + "##∈": 30124, + "##∗": 30125, + "##∘": 30126, + "##√": 30127, + "##∞": 30128, + "##∧": 30129, + "##∨": 30130, + "##∩": 30131, + "##∪": 30132, + "##≈": 30133, + "##≡": 30134, + "##≤": 30135, + "##≥": 30136, + "##⊂": 30137, + "##⊆": 30138, + "##⊕": 30139, + "##⊗": 30140, + "##⋅": 30141, + "##─": 30142, + "##│": 30143, + "##■": 30144, + "##▪": 30145, + "##●": 30146, + "##★": 30147, + "##☆": 30148, + "##☉": 30149, + "##♠": 30150, + "##♣": 30151, + "##♥": 30152, + "##♦": 30153, + "##♯": 30154, + "##⟨": 30155, + "##⟩": 30156, + "##ⱼ": 30157, + "##⺩": 30158, + "##⺼": 30159, + "##⽥": 30160, + "##、": 30161, + "##。": 30162, + "##〈": 30163, + "##〉": 30164, + "##《": 30165, + "##》": 30166, + "##「": 30167, + "##」": 30168, + "##『": 30169, + "##』": 30170, + "##〜": 30171, + "##あ": 30172, + "##い": 30173, + "##う": 30174, + "##え": 30175, + "##お": 30176, + "##か": 30177, + "##き": 30178, + "##く": 30179, + "##け": 30180, + "##こ": 30181, + "##さ": 30182, + "##し": 30183, + "##す": 30184, + "##せ": 30185, + "##そ": 30186, + "##た": 30187, + "##ち": 30188, + "##っ": 30189, + "##つ": 30190, + "##て": 30191, + "##と": 30192, + "##な": 30193, + "##に": 30194, + "##ぬ": 30195, + "##ね": 30196, + "##の": 30197, + "##は": 30198, + "##ひ": 30199, + "##ふ": 30200, + "##へ": 30201, + "##ほ": 30202, + "##ま": 30203, + "##み": 30204, + "##む": 30205, + "##め": 30206, + "##も": 30207, + "##や": 30208, + "##ゆ": 30209, + "##よ": 30210, + "##ら": 30211, + "##り": 30212, + "##る": 30213, + "##れ": 30214, + "##ろ": 30215, + "##を": 30216, + "##ん": 30217, + "##ァ": 30218, + "##ア": 30219, + "##ィ": 30220, + "##イ": 30221, + "##ウ": 30222, + "##ェ": 30223, + "##エ": 30224, + "##オ": 30225, + "##カ": 30226, + "##キ": 30227, + "##ク": 30228, + "##ケ": 30229, + "##コ": 30230, + "##サ": 30231, + "##シ": 30232, + "##ス": 30233, + "##セ": 30234, + "##タ": 30235, + "##チ": 30236, + "##ッ": 30237, + "##ツ": 30238, + "##テ": 30239, + "##ト": 30240, + "##ナ": 30241, + "##ニ": 30242, + "##ノ": 30243, + "##ハ": 30244, + "##ヒ": 30245, + "##フ": 30246, + "##ヘ": 30247, + "##ホ": 30248, + "##マ": 30249, + "##ミ": 30250, + "##ム": 30251, + "##メ": 30252, + "##モ": 30253, + "##ャ": 30254, + "##ュ": 30255, + "##ョ": 30256, + "##ラ": 30257, + "##リ": 30258, + "##ル": 30259, + "##レ": 30260, + "##ロ": 30261, + "##ワ": 30262, + "##ン": 30263, + "##・": 30264, + "##ー": 30265, + "##一": 30266, + "##三": 30267, + "##上": 30268, + "##下": 30269, + "##不": 30270, + "##世": 30271, + "##中": 30272, + "##主": 30273, + "##久": 30274, + "##之": 30275, + "##也": 30276, + "##事": 30277, + "##二": 30278, + "##五": 30279, + "##井": 30280, + "##京": 30281, + "##人": 30282, + "##亻": 30283, + "##仁": 30284, + "##介": 30285, + "##代": 30286, + "##仮": 30287, + "##伊": 30288, + "##会": 30289, + "##佐": 30290, + "##侍": 30291, + "##保": 30292, + "##信": 30293, + "##健": 30294, + "##元": 30295, + "##光": 30296, + "##八": 30297, + "##公": 30298, + "##内": 30299, + "##出": 30300, + "##分": 30301, + "##前": 30302, + "##劉": 30303, + "##力": 30304, + "##加": 30305, + "##勝": 30306, + "##北": 30307, + "##区": 30308, + "##十": 30309, + "##千": 30310, + "##南": 30311, + "##博": 30312, + "##原": 30313, + "##口": 30314, + "##古": 30315, + "##史": 30316, + "##司": 30317, + "##合": 30318, + "##吉": 30319, + "##同": 30320, + "##名": 30321, + "##和": 30322, + "##囗": 30323, + "##四": 30324, + "##国": 30325, + "##國": 30326, + "##土": 30327, + "##地": 30328, + "##坂": 30329, + "##城": 30330, + "##堂": 30331, + "##場": 30332, + "##士": 30333, + "##夏": 30334, + "##外": 30335, + "##大": 30336, + "##天": 30337, + "##太": 30338, + "##夫": 30339, + "##奈": 30340, + "##女": 30341, + "##子": 30342, + "##学": 30343, + "##宀": 30344, + "##宇": 30345, + "##安": 30346, + "##宗": 30347, + "##定": 30348, + "##宣": 30349, + "##宮": 30350, + "##家": 30351, + "##宿": 30352, + "##寺": 30353, + "##將": 30354, + "##小": 30355, + "##尚": 30356, + "##山": 30357, + "##岡": 30358, + "##島": 30359, + "##崎": 30360, + "##川": 30361, + "##州": 30362, + "##巿": 30363, + "##帝": 30364, + "##平": 30365, + "##年": 30366, + "##幸": 30367, + "##广": 30368, + "##弘": 30369, + "##張": 30370, + "##彳": 30371, + "##後": 30372, + "##御": 30373, + "##德": 30374, + "##心": 30375, + "##忄": 30376, + "##志": 30377, + "##忠": 30378, + "##愛": 30379, + "##成": 30380, + "##我": 30381, + "##戦": 30382, + "##戸": 30383, + "##手": 30384, + "##扌": 30385, + "##政": 30386, + "##文": 30387, + "##新": 30388, + "##方": 30389, + "##日": 30390, + "##明": 30391, + "##星": 30392, + "##春": 30393, + "##昭": 30394, + "##智": 30395, + "##曲": 30396, + "##書": 30397, + "##月": 30398, + "##有": 30399, + "##朝": 30400, + "##木": 30401, + "##本": 30402, + "##李": 30403, + "##村": 30404, + "##東": 30405, + "##松": 30406, + "##林": 30407, + "##森": 30408, + "##楊": 30409, + "##樹": 30410, + "##橋": 30411, + "##歌": 30412, + "##止": 30413, + "##正": 30414, + "##武": 30415, + "##比": 30416, + "##氏": 30417, + "##民": 30418, + "##水": 30419, + "##氵": 30420, + "##氷": 30421, + "##永": 30422, + "##江": 30423, + "##沢": 30424, + "##河": 30425, + "##治": 30426, + "##法": 30427, + "##海": 30428, + "##清": 30429, + "##漢": 30430, + "##瀬": 30431, + "##火": 30432, + "##版": 30433, + "##犬": 30434, + "##王": 30435, + "##生": 30436, + "##田": 30437, + "##男": 30438, + "##疒": 30439, + "##発": 30440, + "##白": 30441, + "##的": 30442, + "##皇": 30443, + "##目": 30444, + "##相": 30445, + "##省": 30446, + "##真": 30447, + "##石": 30448, + "##示": 30449, + "##社": 30450, + "##神": 30451, + "##福": 30452, + "##禾": 30453, + "##秀": 30454, + "##秋": 30455, + "##空": 30456, + "##立": 30457, + "##章": 30458, + "##竹": 30459, + "##糹": 30460, + "##美": 30461, + "##義": 30462, + "##耳": 30463, + "##良": 30464, + "##艹": 30465, + "##花": 30466, + "##英": 30467, + "##華": 30468, + "##葉": 30469, + "##藤": 30470, + "##行": 30471, + "##街": 30472, + "##西": 30473, + "##見": 30474, + "##訁": 30475, + "##語": 30476, + "##谷": 30477, + "##貝": 30478, + "##貴": 30479, + "##車": 30480, + "##軍": 30481, + "##辶": 30482, + "##道": 30483, + "##郎": 30484, + "##郡": 30485, + "##部": 30486, + "##都": 30487, + "##里": 30488, + "##野": 30489, + "##金": 30490, + "##鈴": 30491, + "##镇": 30492, + "##長": 30493, + "##門": 30494, + "##間": 30495, + "##阝": 30496, + "##阿": 30497, + "##陳": 30498, + "##陽": 30499, + "##雄": 30500, + "##青": 30501, + "##面": 30502, + "##風": 30503, + "##食": 30504, + "##香": 30505, + "##馬": 30506, + "##高": 30507, + "##龍": 30508, + "##龸": 30509, + "##fi": 30510, + "##fl": 30511, + "##！": 30512, + "##（": 30513, + "##）": 30514, + "##，": 30515, + "##－": 30516, + "##．": 30517, + "##／": 30518, + "##：": 30519, + "##？": 30520, + "##～": 30521 + } + } +} \ No newline at end of file diff --git a/checkpoints/bert-base-uncased/tokenizer_config.json b/checkpoints/bert-base-uncased/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..2a269eff0f34e46722c55f811fca4b295887d635 --- /dev/null +++ b/checkpoints/bert-base-uncased/tokenizer_config.json @@ -0,0 +1,55 @@ +{ + "added_tokens_decoder": { + "0": { + "content": "[PAD]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "100": { + "content": "[UNK]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "101": { + "content": "[CLS]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "102": { + "content": "[SEP]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "103": { + "content": "[MASK]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "clean_up_tokenization_spaces": true, + "cls_token": "[CLS]", + "do_lower_case": true, + "mask_token": "[MASK]", + "model_max_length": 512, + "pad_token": "[PAD]", + "sep_token": "[SEP]", + "strip_accents": null, + "tokenize_chinese_chars": true, + "tokenizer_class": "BertTokenizer", + "unk_token": "[UNK]" +} diff --git a/checkpoints/bert-base-uncased/vocab.txt b/checkpoints/bert-base-uncased/vocab.txt new file mode 100644 index 0000000000000000000000000000000000000000..fb140275c155a9c7c5a3b3e0e77a9e839594a938 --- /dev/null +++ b/checkpoints/bert-base-uncased/vocab.txt @@ -0,0 +1,30522 @@ +[PAD] +[unused0] +[unused1] +[unused2] +[unused3] +[unused4] +[unused5] +[unused6] +[unused7] +[unused8] +[unused9] +[unused10] +[unused11] +[unused12] +[unused13] +[unused14] +[unused15] +[unused16] +[unused17] +[unused18] +[unused19] +[unused20] +[unused21] +[unused22] +[unused23] +[unused24] +[unused25] +[unused26] +[unused27] +[unused28] +[unused29] +[unused30] +[unused31] +[unused32] +[unused33] +[unused34] +[unused35] +[unused36] +[unused37] +[unused38] +[unused39] +[unused40] +[unused41] +[unused42] +[unused43] +[unused44] +[unused45] +[unused46] +[unused47] +[unused48] +[unused49] +[unused50] +[unused51] +[unused52] +[unused53] +[unused54] +[unused55] +[unused56] +[unused57] +[unused58] +[unused59] +[unused60] +[unused61] +[unused62] +[unused63] +[unused64] +[unused65] +[unused66] +[unused67] +[unused68] +[unused69] +[unused70] +[unused71] +[unused72] +[unused73] +[unused74] +[unused75] +[unused76] +[unused77] +[unused78] +[unused79] +[unused80] +[unused81] +[unused82] +[unused83] +[unused84] +[unused85] +[unused86] +[unused87] +[unused88] +[unused89] +[unused90] +[unused91] +[unused92] +[unused93] +[unused94] +[unused95] +[unused96] +[unused97] +[unused98] +[UNK] +[CLS] +[SEP] +[MASK] +[unused99] +[unused100] +[unused101] +[unused102] +[unused103] +[unused104] +[unused105] +[unused106] +[unused107] +[unused108] +[unused109] +[unused110] +[unused111] +[unused112] +[unused113] +[unused114] +[unused115] +[unused116] +[unused117] +[unused118] +[unused119] +[unused120] +[unused121] +[unused122] +[unused123] +[unused124] +[unused125] +[unused126] +[unused127] +[unused128] +[unused129] +[unused130] +[unused131] +[unused132] +[unused133] +[unused134] +[unused135] +[unused136] +[unused137] +[unused138] +[unused139] +[unused140] +[unused141] +[unused142] +[unused143] +[unused144] +[unused145] +[unused146] +[unused147] +[unused148] +[unused149] +[unused150] +[unused151] +[unused152] +[unused153] +[unused154] +[unused155] +[unused156] +[unused157] +[unused158] +[unused159] +[unused160] +[unused161] +[unused162] +[unused163] +[unused164] +[unused165] +[unused166] +[unused167] +[unused168] +[unused169] +[unused170] +[unused171] +[unused172] +[unused173] +[unused174] +[unused175] +[unused176] +[unused177] +[unused178] +[unused179] +[unused180] +[unused181] +[unused182] +[unused183] +[unused184] +[unused185] +[unused186] +[unused187] +[unused188] +[unused189] +[unused190] +[unused191] +[unused192] +[unused193] +[unused194] +[unused195] +[unused196] +[unused197] +[unused198] +[unused199] +[unused200] +[unused201] +[unused202] +[unused203] +[unused204] +[unused205] +[unused206] +[unused207] +[unused208] +[unused209] +[unused210] +[unused211] +[unused212] +[unused213] +[unused214] +[unused215] +[unused216] +[unused217] +[unused218] +[unused219] +[unused220] +[unused221] +[unused222] +[unused223] +[unused224] +[unused225] +[unused226] +[unused227] +[unused228] +[unused229] +[unused230] +[unused231] +[unused232] +[unused233] +[unused234] +[unused235] +[unused236] +[unused237] +[unused238] +[unused239] +[unused240] +[unused241] +[unused242] +[unused243] +[unused244] +[unused245] +[unused246] +[unused247] +[unused248] +[unused249] +[unused250] +[unused251] +[unused252] +[unused253] +[unused254] +[unused255] +[unused256] +[unused257] +[unused258] +[unused259] +[unused260] +[unused261] +[unused262] +[unused263] +[unused264] +[unused265] +[unused266] +[unused267] +[unused268] +[unused269] +[unused270] +[unused271] +[unused272] +[unused273] +[unused274] +[unused275] +[unused276] +[unused277] +[unused278] +[unused279] +[unused280] +[unused281] +[unused282] +[unused283] +[unused284] +[unused285] +[unused286] +[unused287] +[unused288] +[unused289] +[unused290] +[unused291] +[unused292] +[unused293] +[unused294] +[unused295] +[unused296] +[unused297] +[unused298] +[unused299] +[unused300] +[unused301] +[unused302] +[unused303] +[unused304] +[unused305] +[unused306] +[unused307] +[unused308] +[unused309] +[unused310] +[unused311] +[unused312] +[unused313] +[unused314] +[unused315] +[unused316] +[unused317] +[unused318] +[unused319] +[unused320] +[unused321] +[unused322] +[unused323] +[unused324] +[unused325] +[unused326] +[unused327] +[unused328] +[unused329] +[unused330] +[unused331] +[unused332] +[unused333] +[unused334] +[unused335] +[unused336] +[unused337] +[unused338] +[unused339] +[unused340] +[unused341] +[unused342] +[unused343] +[unused344] +[unused345] +[unused346] +[unused347] +[unused348] +[unused349] +[unused350] +[unused351] +[unused352] +[unused353] +[unused354] +[unused355] +[unused356] +[unused357] +[unused358] +[unused359] +[unused360] +[unused361] +[unused362] +[unused363] +[unused364] +[unused365] +[unused366] +[unused367] +[unused368] +[unused369] +[unused370] +[unused371] +[unused372] +[unused373] +[unused374] +[unused375] +[unused376] +[unused377] +[unused378] +[unused379] +[unused380] +[unused381] +[unused382] +[unused383] +[unused384] +[unused385] +[unused386] +[unused387] +[unused388] +[unused389] +[unused390] +[unused391] +[unused392] +[unused393] +[unused394] +[unused395] +[unused396] +[unused397] +[unused398] +[unused399] +[unused400] +[unused401] +[unused402] +[unused403] +[unused404] +[unused405] +[unused406] +[unused407] +[unused408] +[unused409] +[unused410] +[unused411] +[unused412] +[unused413] +[unused414] +[unused415] +[unused416] +[unused417] +[unused418] +[unused419] +[unused420] +[unused421] +[unused422] +[unused423] +[unused424] +[unused425] +[unused426] +[unused427] +[unused428] +[unused429] +[unused430] +[unused431] +[unused432] +[unused433] +[unused434] +[unused435] +[unused436] +[unused437] +[unused438] +[unused439] +[unused440] +[unused441] +[unused442] +[unused443] +[unused444] +[unused445] +[unused446] +[unused447] +[unused448] +[unused449] +[unused450] +[unused451] +[unused452] +[unused453] +[unused454] +[unused455] +[unused456] +[unused457] +[unused458] +[unused459] +[unused460] +[unused461] +[unused462] +[unused463] +[unused464] +[unused465] +[unused466] +[unused467] +[unused468] +[unused469] +[unused470] +[unused471] +[unused472] +[unused473] +[unused474] +[unused475] +[unused476] +[unused477] +[unused478] +[unused479] +[unused480] +[unused481] +[unused482] +[unused483] +[unused484] +[unused485] +[unused486] +[unused487] +[unused488] +[unused489] +[unused490] +[unused491] +[unused492] +[unused493] +[unused494] +[unused495] +[unused496] +[unused497] +[unused498] +[unused499] +[unused500] +[unused501] +[unused502] +[unused503] +[unused504] +[unused505] +[unused506] +[unused507] +[unused508] +[unused509] +[unused510] +[unused511] +[unused512] +[unused513] +[unused514] +[unused515] +[unused516] +[unused517] +[unused518] +[unused519] +[unused520] +[unused521] +[unused522] +[unused523] +[unused524] +[unused525] +[unused526] +[unused527] +[unused528] +[unused529] +[unused530] +[unused531] +[unused532] +[unused533] +[unused534] +[unused535] +[unused536] +[unused537] +[unused538] +[unused539] +[unused540] +[unused541] +[unused542] +[unused543] +[unused544] +[unused545] +[unused546] +[unused547] +[unused548] +[unused549] +[unused550] +[unused551] +[unused552] +[unused553] +[unused554] +[unused555] +[unused556] +[unused557] +[unused558] +[unused559] +[unused560] +[unused561] +[unused562] +[unused563] +[unused564] +[unused565] +[unused566] +[unused567] +[unused568] +[unused569] +[unused570] +[unused571] +[unused572] +[unused573] +[unused574] +[unused575] +[unused576] +[unused577] +[unused578] +[unused579] +[unused580] +[unused581] +[unused582] +[unused583] +[unused584] +[unused585] +[unused586] +[unused587] +[unused588] +[unused589] +[unused590] +[unused591] +[unused592] +[unused593] +[unused594] +[unused595] +[unused596] +[unused597] +[unused598] +[unused599] +[unused600] +[unused601] +[unused602] +[unused603] +[unused604] +[unused605] +[unused606] +[unused607] +[unused608] +[unused609] +[unused610] +[unused611] +[unused612] +[unused613] +[unused614] +[unused615] +[unused616] +[unused617] +[unused618] +[unused619] +[unused620] +[unused621] +[unused622] +[unused623] +[unused624] +[unused625] +[unused626] +[unused627] +[unused628] +[unused629] +[unused630] +[unused631] +[unused632] +[unused633] +[unused634] +[unused635] +[unused636] +[unused637] +[unused638] +[unused639] +[unused640] +[unused641] +[unused642] +[unused643] +[unused644] +[unused645] +[unused646] +[unused647] +[unused648] +[unused649] +[unused650] +[unused651] +[unused652] +[unused653] +[unused654] +[unused655] +[unused656] +[unused657] +[unused658] +[unused659] +[unused660] +[unused661] +[unused662] +[unused663] +[unused664] +[unused665] +[unused666] +[unused667] +[unused668] +[unused669] +[unused670] +[unused671] +[unused672] +[unused673] +[unused674] +[unused675] +[unused676] +[unused677] +[unused678] +[unused679] +[unused680] +[unused681] +[unused682] +[unused683] +[unused684] +[unused685] +[unused686] +[unused687] +[unused688] +[unused689] +[unused690] +[unused691] +[unused692] +[unused693] +[unused694] +[unused695] +[unused696] +[unused697] +[unused698] +[unused699] +[unused700] +[unused701] +[unused702] +[unused703] +[unused704] +[unused705] +[unused706] +[unused707] +[unused708] +[unused709] +[unused710] +[unused711] +[unused712] +[unused713] +[unused714] +[unused715] +[unused716] +[unused717] +[unused718] +[unused719] +[unused720] +[unused721] +[unused722] +[unused723] +[unused724] +[unused725] +[unused726] +[unused727] +[unused728] +[unused729] +[unused730] +[unused731] +[unused732] +[unused733] +[unused734] +[unused735] +[unused736] +[unused737] +[unused738] +[unused739] +[unused740] +[unused741] +[unused742] +[unused743] +[unused744] +[unused745] +[unused746] +[unused747] +[unused748] +[unused749] +[unused750] +[unused751] +[unused752] +[unused753] +[unused754] +[unused755] +[unused756] +[unused757] +[unused758] +[unused759] +[unused760] +[unused761] +[unused762] +[unused763] +[unused764] +[unused765] +[unused766] +[unused767] +[unused768] +[unused769] +[unused770] +[unused771] +[unused772] +[unused773] +[unused774] +[unused775] +[unused776] +[unused777] +[unused778] +[unused779] +[unused780] +[unused781] +[unused782] +[unused783] +[unused784] +[unused785] +[unused786] +[unused787] +[unused788] +[unused789] +[unused790] +[unused791] +[unused792] +[unused793] +[unused794] +[unused795] +[unused796] +[unused797] +[unused798] +[unused799] +[unused800] +[unused801] +[unused802] +[unused803] +[unused804] +[unused805] +[unused806] +[unused807] +[unused808] +[unused809] +[unused810] +[unused811] +[unused812] +[unused813] +[unused814] +[unused815] +[unused816] +[unused817] +[unused818] +[unused819] +[unused820] +[unused821] +[unused822] +[unused823] +[unused824] +[unused825] +[unused826] +[unused827] +[unused828] +[unused829] +[unused830] +[unused831] +[unused832] +[unused833] +[unused834] +[unused835] +[unused836] +[unused837] +[unused838] +[unused839] +[unused840] +[unused841] +[unused842] +[unused843] +[unused844] +[unused845] +[unused846] +[unused847] +[unused848] +[unused849] +[unused850] +[unused851] +[unused852] +[unused853] +[unused854] +[unused855] +[unused856] +[unused857] +[unused858] +[unused859] +[unused860] +[unused861] +[unused862] +[unused863] +[unused864] +[unused865] +[unused866] +[unused867] +[unused868] +[unused869] +[unused870] +[unused871] +[unused872] +[unused873] +[unused874] +[unused875] +[unused876] +[unused877] +[unused878] +[unused879] +[unused880] +[unused881] +[unused882] +[unused883] +[unused884] +[unused885] +[unused886] +[unused887] +[unused888] +[unused889] +[unused890] +[unused891] +[unused892] +[unused893] +[unused894] +[unused895] +[unused896] +[unused897] +[unused898] +[unused899] +[unused900] +[unused901] +[unused902] +[unused903] +[unused904] +[unused905] +[unused906] +[unused907] +[unused908] +[unused909] +[unused910] +[unused911] +[unused912] +[unused913] +[unused914] +[unused915] +[unused916] +[unused917] +[unused918] +[unused919] +[unused920] +[unused921] +[unused922] +[unused923] +[unused924] +[unused925] +[unused926] +[unused927] +[unused928] +[unused929] +[unused930] +[unused931] +[unused932] +[unused933] +[unused934] +[unused935] +[unused936] +[unused937] +[unused938] +[unused939] +[unused940] +[unused941] +[unused942] +[unused943] +[unused944] +[unused945] +[unused946] +[unused947] +[unused948] +[unused949] +[unused950] +[unused951] +[unused952] +[unused953] +[unused954] +[unused955] +[unused956] +[unused957] +[unused958] +[unused959] +[unused960] +[unused961] +[unused962] +[unused963] +[unused964] +[unused965] +[unused966] +[unused967] +[unused968] +[unused969] +[unused970] +[unused971] +[unused972] +[unused973] +[unused974] +[unused975] +[unused976] +[unused977] +[unused978] +[unused979] +[unused980] +[unused981] +[unused982] +[unused983] +[unused984] +[unused985] +[unused986] +[unused987] +[unused988] +[unused989] +[unused990] +[unused991] +[unused992] +[unused993] +! +" +# +$ +% +& +' +( +) +* ++ +, +- +. +/ +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +: +; +< += +> +? +@ +[ +\ +] +^ +_ +` +a +b +c +d +e +f +g +h +i +j +k +l +m +n +o +p +q +r +s +t +u +v +w +x +y +z +{ +| +} +~ +¡ +¢ +£ +¤ +¥ +¦ +§ +¨ +© +ª +« +¬ +® +° +± +² +³ +´ +µ +¶ +· +¹ +º +» +¼ +½ +¾ +¿ +× +ß +æ +ð +÷ +ø +þ +đ +ħ +ı +ł +ŋ +œ +ƒ +ɐ +ɑ +ɒ +ɔ +ɕ +ə +ɛ +ɡ +ɣ +ɨ +ɪ +ɫ +ɬ +ɯ +ɲ +ɴ +ɹ +ɾ +ʀ +ʁ +ʂ +ʃ +ʉ +ʊ +ʋ +ʌ +ʎ +ʐ +ʑ +ʒ +ʔ +ʰ +ʲ +ʳ +ʷ +ʸ +ʻ +ʼ +ʾ +ʿ +ˈ +ː +ˡ +ˢ +ˣ +ˤ +α +β +γ +δ +ε +ζ +η +θ +ι +κ +λ +μ +ν +ξ +ο +π +ρ +ς +σ +τ +υ +φ +χ +ψ +ω +а +б +в +г +д +е +ж +з +и +к +л +м +н +о +п +р +с +т +у +ф +х +ц +ч +ш +щ +ъ +ы +ь +э +ю +я +ђ +є +і +ј +љ +њ +ћ +ӏ +ա +բ +գ +դ +ե +թ +ի +լ +կ +հ +մ +յ +ն +ո +պ +ս +վ +տ +ր +ւ +ք +־ +א +ב +ג +ד +ה +ו +ז +ח +ט +י +ך +כ +ל +ם +מ +ן +נ +ס +ע +ף +פ +ץ +צ +ק +ר +ש +ת +، +ء +ا +ب +ة +ت +ث +ج +ح +خ +د +ذ +ر +ز +س +ش +ص +ض +ط +ظ +ع +غ +ـ +ف +ق +ك +ل +م +ن +ه +و +ى +ي +ٹ +پ +چ +ک +گ +ں +ھ +ہ +ی +ے +अ +आ +उ +ए +क +ख +ग +च +ज +ट +ड +ण +त +थ +द +ध +न +प +ब +भ +म +य +र +ल +व +श +ष +स +ह +ा +ि +ी +ो +। +॥ +ং +অ +আ +ই +উ +এ +ও +ক +খ +গ +চ +ছ +জ +ট +ড +ণ +ত +থ +দ +ধ +ন +প +ব +ভ +ম +য +র +ল +শ +ষ +স +হ +া +ি +ী +ে +க +ச +ட +த +ந +ன +ப +ம +ய +ர +ல +ள +வ +ா +ி +ு +ே +ை +ನ +ರ +ಾ +ක +ය +ර +ල +ව +ා +ก +ง +ต +ท +น +พ +ม +ย +ร +ล +ว +ส +อ +า +เ +་ +། +ག +ང +ད +ན +པ +བ +མ +འ +ར +ལ +ས +မ +ა +ბ +გ +დ +ე +ვ +თ +ი +კ +ლ +მ +ნ +ო +რ +ს +ტ +უ +ᄀ +ᄂ +ᄃ +ᄅ +ᄆ +ᄇ +ᄉ +ᄊ +ᄋ +ᄌ +ᄎ +ᄏ +ᄐ +ᄑ +ᄒ +ᅡ +ᅢ +ᅥ +ᅦ +ᅧ +ᅩ +ᅪ +ᅭ +ᅮ +ᅯ +ᅲ +ᅳ +ᅴ +ᅵ +ᆨ +ᆫ +ᆯ +ᆷ +ᆸ +ᆼ +ᴬ +ᴮ +ᴰ +ᴵ +ᴺ +ᵀ +ᵃ +ᵇ +ᵈ +ᵉ +ᵍ +ᵏ +ᵐ +ᵒ +ᵖ +ᵗ +ᵘ +ᵢ +ᵣ +ᵤ +ᵥ +ᶜ +ᶠ +‐ +‑ +‒ +– +— +― +‖ +‘ +’ +‚ +“ +” +„ +† +‡ +• +… +‰ +′ +″ +› +‿ +⁄ +⁰ +ⁱ +⁴ +⁵ +⁶ +⁷ +⁸ +⁹ +⁺ +⁻ +ⁿ +₀ +₁ +₂ +₃ +₄ +₅ +₆ +₇ +₈ +₉ +₊ +₍ +₎ +ₐ +ₑ +ₒ +ₓ +ₕ +ₖ +ₗ +ₘ +ₙ +ₚ +ₛ +ₜ +₤ +₩ +€ +₱ +₹ +ℓ +№ +ℝ +™ +⅓ +⅔ +← +↑ +→ +↓ +↔ +↦ +⇄ +⇌ +⇒ +∂ +∅ +∆ +∇ +∈ +− +∗ +∘ +√ +∞ +∧ +∨ +∩ +∪ +≈ +≡ +≤ +≥ +⊂ +⊆ +⊕ +⊗ +⋅ +─ +│ +■ +▪ +● +★ +☆ +☉ +♠ +♣ +♥ +♦ +♭ +♯ +⟨ +⟩ +ⱼ +⺩ +⺼ +⽥ +、 +。 +〈 +〉 +《 +》 +「 +」 +『 +』 +〜 +あ +い +う +え +お +か +き +く +け +こ +さ +し +す +せ +そ +た +ち +っ +つ +て +と +な +に +ぬ +ね +の +は +ひ +ふ +へ +ほ +ま +み +む +め +も +や +ゆ +よ +ら +り +る +れ +ろ +を +ん +ァ +ア +ィ +イ +ウ +ェ +エ +オ +カ +キ +ク +ケ +コ +サ +シ +ス +セ +タ +チ +ッ +ツ +テ +ト +ナ +ニ +ノ +ハ +ヒ +フ +ヘ +ホ +マ +ミ +ム +メ +モ +ャ +ュ +ョ +ラ +リ +ル +レ +ロ +ワ +ン +・ +ー +一 +三 +上 +下 +不 +世 +中 +主 +久 +之 +也 +事 +二 +五 +井 +京 +人 +亻 +仁 +介 +代 +仮 +伊 +会 +佐 +侍 +保 +信 +健 +元 +光 +八 +公 +内 +出 +分 +前 +劉 +力 +加 +勝 +北 +区 +十 +千 +南 +博 +原 +口 +古 +史 +司 +合 +吉 +同 +名 +和 +囗 +四 +国 +國 +土 +地 +坂 +城 +堂 +場 +士 +夏 +外 +大 +天 +太 +夫 +奈 +女 +子 +学 +宀 +宇 +安 +宗 +定 +宣 +宮 +家 +宿 +寺 +將 +小 +尚 +山 +岡 +島 +崎 +川 +州 +巿 +帝 +平 +年 +幸 +广 +弘 +張 +彳 +後 +御 +德 +心 +忄 +志 +忠 +愛 +成 +我 +戦 +戸 +手 +扌 +政 +文 +新 +方 +日 +明 +星 +春 +昭 +智 +曲 +書 +月 +有 +朝 +木 +本 +李 +村 +東 +松 +林 +森 +楊 +樹 +橋 +歌 +止 +正 +武 +比 +氏 +民 +水 +氵 +氷 +永 +江 +沢 +河 +治 +法 +海 +清 +漢 +瀬 +火 +版 +犬 +王 +生 +田 +男 +疒 +発 +白 +的 +皇 +目 +相 +省 +真 +石 +示 +社 +神 +福 +禾 +秀 +秋 +空 +立 +章 +竹 +糹 +美 +義 +耳 +良 +艹 +花 +英 +華 +葉 +藤 +行 +街 +西 +見 +訁 +語 +谷 +貝 +貴 +車 +軍 +辶 +道 +郎 +郡 +部 +都 +里 +野 +金 +鈴 +镇 +長 +門 +間 +阝 +阿 +陳 +陽 +雄 +青 +面 +風 +食 +香 +馬 +高 +龍 +龸 +fi +fl +！ +（ +） +， +－ +． +／ +： +？ +～ +the +of +and +in +to +was +he +is +as +for +on +with +that +it +his +by +at +from +her +##s +she +you +had +an +were +but +be +this +are +not +my +they +one +which +or +have +him +me +first +all +also +their +has +up +who +out +been +when +after +there +into +new +two +its +##a +time +would +no +what +about +said +we +over +then +other +so +more +##e +can +if +like +back +them +only +some +could +##i +where +just +##ing +during +before +##n +do +##o +made +school +through +than +now +years +most +world +may +between +down +well +three +##d +year +while +will +##ed +##r +##y +later +##t +city +under +around +did +such +being +used +state +people +part +know +against +your +many +second +university +both +national +##er +these +don +known +off +way +until +re +how +even +get +head +... +didn +##ly +team +american +because +de +##l +born +united +film +since +still +long +work +south +us +became +any +high +again +day +family +see +right +man +eyes +house +season +war +states +including +took +life +north +same +each +called +name +much +place +however +go +four +group +another +found +won +area +here +going +10 +away +series +left +home +music +best +make +hand +number +company +several +never +last +john +000 +very +album +take +end +good +too +following +released +game +played +little +began +district +##m +old +want +those +side +held +own +early +county +ll +league +use +west +##u +face +think +##es +2010 +government +##h +march +came +small +general +town +june +##on +line +based +something +##k +september +thought +looked +along +international +2011 +air +july +club +went +january +october +our +august +april +york +12 +few +2012 +2008 +east +show +member +college +2009 +father +public +##us +come +men +five +set +station +church +##c +next +former +november +room +party +located +december +2013 +age +got +2007 +##g +system +let +love +2006 +though +every +2014 +look +song +water +century +without +body +black +night +within +great +women +single +ve +building +large +population +river +named +band +white +started +##an +once +15 +20 +should +18 +2015 +service +top +built +british +open +death +king +moved +local +times +children +february +book +why +11 +door +need +president +order +final +road +wasn +although +due +major +died +village +third +knew +2016 +asked +turned +st +wanted +say +##p +together +received +main +son +served +different +##en +behind +himself +felt +members +power +football +law +voice +play +##in +near +park +history +30 +having +2005 +16 +##man +saw +mother +##al +army +point +front +help +english +street +art +late +hands +games +award +##ia +young +14 +put +published +country +division +across +told +13 +often +ever +french +london +center +six +red +2017 +led +days +include +light +25 +find +tell +among +species +really +according +central +half +2004 +form +original +gave +office +making +enough +lost +full +opened +must +included +live +given +german +player +run +business +woman +community +cup +might +million +land +2000 +court +development +17 +short +round +ii +km +seen +class +story +always +become +sure +research +almost +director +council +la +##2 +career +things +using +island +##z +couldn +car +##is +24 +close +force +##1 +better +free +support +control +field +students +2003 +education +married +##b +nothing +worked +others +record +big +inside +level +anything +continued +give +james +##3 +military +established +non +returned +feel +does +title +written +thing +feet +william +far +co +association +hard +already +2002 +##ra +championship +human +western +100 +##na +department +hall +role +various +production +21 +19 +heart +2001 +living +fire +version +##ers +##f +television +royal +##4 +produced +working +act +case +society +region +present +radio +period +looking +least +total +keep +england +wife +program +per +brother +mind +special +22 +##le +am +works +soon +##6 +political +george +services +taken +created +##7 +further +able +reached +david +union +joined +upon +done +important +social +information +either +##ic +##x +appeared +position +ground +lead +rock +dark +election +23 +board +france +hair +course +arms +site +police +girl +instead +real +sound +##v +words +moment +##te +someone +##8 +summer +project +announced +san +less +wrote +past +followed +##5 +blue +founded +al +finally +india +taking +records +america +##ne +1999 +design +considered +northern +god +stop +battle +toward +european +outside +described +track +today +playing +language +28 +call +26 +heard +professional +low +australia +miles +california +win +yet +green +##ie +trying +blood +##ton +southern +science +maybe +everything +match +square +27 +mouth +video +race +recorded +leave +above +##9 +daughter +points +space +1998 +museum +change +middle +common +##0 +move +tv +post +##ta +lake +seven +tried +elected +closed +ten +paul +minister +##th +months +start +chief +return +canada +person +sea +release +similar +modern +brought +rest +hit +formed +mr +##la +1997 +floor +event +doing +thomas +1996 +robert +care +killed +training +star +week +needed +turn +finished +railway +rather +news +health +sent +example +ran +term +michael +coming +currently +yes +forces +despite +gold +areas +50 +stage +fact +29 +dead +says +popular +2018 +originally +germany +probably +developed +result +pulled +friend +stood +money +running +mi +signed +word +songs +child +eventually +met +tour +average +teams +minutes +festival +current +deep +kind +1995 +decided +usually +eastern +seemed +##ness +episode +bed +added +table +indian +private +charles +route +available +idea +throughout +centre +addition +appointed +style +1994 +books +eight +construction +press +mean +wall +friends +remained +schools +study +##ch +##um +institute +oh +chinese +sometimes +events +possible +1992 +australian +type +brown +forward +talk +process +food +debut +seat +performance +committee +features +character +arts +herself +else +lot +strong +russian +range +hours +peter +arm +##da +morning +dr +sold +##ry +quickly +directed +1993 +guitar +china +##w +31 +list +##ma +performed +media +uk +players +smile +##rs +myself +40 +placed +coach +province +towards +wouldn +leading +whole +boy +official +designed +grand +census +##el +europe +attack +japanese +henry +1991 +##re +##os +cross +getting +alone +action +lower +network +wide +washington +japan +1990 +hospital +believe +changed +sister +##ar +hold +gone +sir +hadn +ship +##ka +studies +academy +shot +rights +below +base +bad +involved +kept +largest +##ist +bank +future +especially +beginning +mark +movement +section +female +magazine +plan +professor +lord +longer +##ian +sat +walked +hill +actually +civil +energy +model +families +size +thus +aircraft +completed +includes +data +captain +##or +fight +vocals +featured +richard +bridge +fourth +1989 +officer +stone +hear +##ism +means +medical +groups +management +self +lips +competition +entire +lived +technology +leaving +federal +tournament +bit +passed +hot +independent +awards +kingdom +mary +spent +fine +doesn +reported +##ling +jack +fall +raised +itself +stay +true +studio +1988 +sports +replaced +paris +systems +saint +leader +theatre +whose +market +capital +parents +spanish +canadian +earth +##ity +cut +degree +writing +bay +christian +awarded +natural +higher +bill +##as +coast +provided +previous +senior +ft +valley +organization +stopped +onto +countries +parts +conference +queen +security +interest +saying +allowed +master +earlier +phone +matter +smith +winning +try +happened +moving +campaign +los +##ley +breath +nearly +mid +1987 +certain +girls +date +italian +african +standing +fell +artist +##ted +shows +deal +mine +industry +1986 +##ng +everyone +republic +provide +collection +library +student +##ville +primary +owned +older +via +heavy +1st +makes +##able +attention +anyone +africa +##ri +stated +length +ended +fingers +command +staff +skin +foreign +opening +governor +okay +medal +kill +sun +cover +job +1985 +introduced +chest +hell +feeling +##ies +success +meet +reason +standard +meeting +novel +1984 +trade +source +buildings +##land +rose +guy +goal +##ur +chapter +native +husband +previously +unit +limited +entered +weeks +producer +operations +mountain +takes +covered +forced +related +roman +complete +successful +key +texas +cold +##ya +channel +1980 +traditional +films +dance +clear +approximately +500 +nine +van +prince +question +active +tracks +ireland +regional +silver +author +personal +sense +operation +##ine +economic +1983 +holding +twenty +isbn +additional +speed +hour +edition +regular +historic +places +whom +shook +movie +km² +secretary +prior +report +chicago +read +foundation +view +engine +scored +1982 +units +ask +airport +property +ready +immediately +lady +month +listed +contract +##de +manager +themselves +lines +##ki +navy +writer +meant +##ts +runs +##ro +practice +championships +singer +glass +commission +required +forest +starting +culture +generally +giving +access +attended +test +couple +stand +catholic +martin +caught +executive +##less +eye +##ey +thinking +chair +quite +shoulder +1979 +hope +decision +plays +defeated +municipality +whether +structure +offered +slowly +pain +ice +direction +##ion +paper +mission +1981 +mostly +200 +noted +individual +managed +nature +lives +plant +##ha +helped +except +studied +computer +figure +relationship +issue +significant +loss +die +smiled +gun +ago +highest +1972 +##am +male +bring +goals +mexico +problem +distance +commercial +completely +location +annual +famous +drive +1976 +neck +1978 +surface +caused +italy +understand +greek +highway +wrong +hotel +comes +appearance +joseph +double +issues +musical +companies +castle +income +review +assembly +bass +initially +parliament +artists +experience +1974 +particular +walk +foot +engineering +talking +window +dropped +##ter +miss +baby +boys +break +1975 +stars +edge +remember +policy +carried +train +stadium +bar +sex +angeles +evidence +##ge +becoming +assistant +soviet +1977 +upper +step +wing +1970 +youth +financial +reach +##ll +actor +numerous +##se +##st +nodded +arrived +##ation +minute +##nt +believed +sorry +complex +beautiful +victory +associated +temple +1968 +1973 +chance +perhaps +metal +##son +1945 +bishop +##et +lee +launched +particularly +tree +le +retired +subject +prize +contains +yeah +theory +empire +##ce +suddenly +waiting +trust +recording +##to +happy +terms +camp +champion +1971 +religious +pass +zealand +names +2nd +port +ancient +tom +corner +represented +watch +legal +anti +justice +cause +watched +brothers +45 +material +changes +simply +response +louis +fast +##ting +answer +60 +historical +1969 +stories +straight +create +feature +increased +rate +administration +virginia +el +activities +cultural +overall +winner +programs +basketball +legs +guard +beyond +cast +doctor +mm +flight +results +remains +cost +effect +winter +##ble +larger +islands +problems +chairman +grew +commander +isn +1967 +pay +failed +selected +hurt +fort +box +regiment +majority +journal +35 +edward +plans +##ke +##ni +shown +pretty +irish +characters +directly +scene +likely +operated +allow +spring +##j +junior +matches +looks +mike +houses +fellow +##tion +beach +marriage +##ham +##ive +rules +oil +65 +florida +expected +nearby +congress +sam +peace +recent +iii +wait +subsequently +cell +##do +variety +serving +agreed +please +poor +joe +pacific +attempt +wood +democratic +piece +prime +##ca +rural +mile +touch +appears +township +1964 +1966 +soldiers +##men +##ized +1965 +pennsylvania +closer +fighting +claimed +score +jones +physical +editor +##ous +filled +genus +specific +sitting +super +mom +##va +therefore +supported +status +fear +cases +store +meaning +wales +minor +spain +tower +focus +vice +frank +follow +parish +separate +golden +horse +fifth +remaining +branch +32 +presented +stared +##id +uses +secret +forms +##co +baseball +exactly +##ck +choice +note +discovered +travel +composed +truth +russia +ball +color +kiss +dad +wind +continue +ring +referred +numbers +digital +greater +##ns +metres +slightly +direct +increase +1960 +responsible +crew +rule +trees +troops +##no +broke +goes +individuals +hundred +weight +creek +sleep +memory +defense +provides +ordered +code +value +jewish +windows +1944 +safe +judge +whatever +corps +realized +growing +pre +##ga +cities +alexander +gaze +lies +spread +scott +letter +showed +situation +mayor +transport +watching +workers +extended +##li +expression +normal +##ment +chart +multiple +border +##ba +host +##ner +daily +mrs +walls +piano +##ko +heat +cannot +##ate +earned +products +drama +era +authority +seasons +join +grade +##io +sign +difficult +machine +1963 +territory +mainly +##wood +stations +squadron +1962 +stepped +iron +19th +##led +serve +appear +sky +speak +broken +charge +knowledge +kilometres +removed +ships +article +campus +simple +##ty +pushed +britain +##ve +leaves +recently +cd +soft +boston +latter +easy +acquired +poland +##sa +quality +officers +presence +planned +nations +mass +broadcast +jean +share +image +influence +wild +offer +emperor +electric +reading +headed +ability +promoted +yellow +ministry +1942 +throat +smaller +politician +##by +latin +spoke +cars +williams +males +lack +pop +80 +##ier +acting +seeing +consists +##ti +estate +1961 +pressure +johnson +newspaper +jr +chris +olympics +online +conditions +beat +elements +walking +vote +##field +needs +carolina +text +featuring +global +block +shirt +levels +francisco +purpose +females +et +dutch +duke +ahead +gas +twice +safety +serious +turning +highly +lieutenant +firm +maria +amount +mixed +daniel +proposed +perfect +agreement +affairs +3rd +seconds +contemporary +paid +1943 +prison +save +kitchen +label +administrative +intended +constructed +academic +nice +teacher +races +1956 +formerly +corporation +ben +nation +issued +shut +1958 +drums +housing +victoria +seems +opera +1959 +graduated +function +von +mentioned +picked +build +recognized +shortly +protection +picture +notable +exchange +elections +1980s +loved +percent +racing +fish +elizabeth +garden +volume +hockey +1941 +beside +settled +##ford +1940 +competed +replied +drew +1948 +actress +marine +scotland +steel +glanced +farm +steve +1957 +risk +tonight +positive +magic +singles +effects +gray +screen +dog +##ja +residents +bus +sides +none +secondary +literature +polish +destroyed +flying +founder +households +1939 +lay +reserve +usa +gallery +##ler +1946 +industrial +younger +approach +appearances +urban +ones +1950 +finish +avenue +powerful +fully +growth +page +honor +jersey +projects +advanced +revealed +basic +90 +infantry +pair +equipment +visit +33 +evening +search +grant +effort +solo +treatment +buried +republican +primarily +bottom +owner +1970s +israel +gives +jim +dream +bob +remain +spot +70 +notes +produce +champions +contact +ed +soul +accepted +ways +del +##ally +losing +split +price +capacity +basis +trial +questions +##ina +1955 +20th +guess +officially +memorial +naval +initial +##ization +whispered +median +engineer +##ful +sydney +##go +columbia +strength +300 +1952 +tears +senate +00 +card +asian +agent +1947 +software +44 +draw +warm +supposed +com +pro +##il +transferred +leaned +##at +candidate +escape +mountains +asia +potential +activity +entertainment +seem +traffic +jackson +murder +36 +slow +product +orchestra +haven +agency +bbc +taught +website +comedy +unable +storm +planning +albums +rugby +environment +scientific +grabbed +protect +##hi +boat +typically +1954 +1953 +damage +principal +divided +dedicated +mount +ohio +##berg +pick +fought +driver +##der +empty +shoulders +sort +thank +berlin +prominent +account +freedom +necessary +efforts +alex +headquarters +follows +alongside +des +simon +andrew +suggested +operating +learning +steps +1949 +sweet +technical +begin +easily +34 +teeth +speaking +settlement +scale +##sh +renamed +ray +max +enemy +semi +joint +compared +##rd +scottish +leadership +analysis +offers +georgia +pieces +captured +animal +deputy +guest +organized +##lin +tony +combined +method +challenge +1960s +huge +wants +battalion +sons +rise +crime +types +facilities +telling +path +1951 +platform +sit +1990s +##lo +tells +assigned +rich +pull +##ot +commonly +alive +##za +letters +concept +conducted +wearing +happen +bought +becomes +holy +gets +ocean +defeat +languages +purchased +coffee +occurred +titled +##q +declared +applied +sciences +concert +sounds +jazz +brain +##me +painting +fleet +tax +nick +##ius +michigan +count +animals +leaders +episodes +##line +content +##den +birth +##it +clubs +64 +palace +critical +refused +fair +leg +laughed +returning +surrounding +participated +formation +lifted +pointed +connected +rome +medicine +laid +taylor +santa +powers +adam +tall +shared +focused +knowing +yards +entrance +falls +##wa +calling +##ad +sources +chosen +beneath +resources +yard +##ite +nominated +silence +zone +defined +##que +gained +thirty +38 +bodies +moon +##ard +adopted +christmas +widely +register +apart +iran +premier +serves +du +unknown +parties +##les +generation +##ff +continues +quick +fields +brigade +quiet +teaching +clothes +impact +weapons +partner +flat +theater +supreme +1938 +37 +relations +##tor +plants +suffered +1936 +wilson +kids +begins +##age +1918 +seats +armed +internet +models +worth +laws +400 +communities +classes +background +knows +thanks +quarter +reaching +humans +carry +killing +format +kong +hong +setting +75 +architecture +disease +railroad +inc +possibly +wish +arthur +thoughts +harry +doors +density +##di +crowd +illinois +stomach +tone +unique +reports +anyway +##ir +liberal +der +vehicle +thick +dry +drug +faced +largely +facility +theme +holds +creation +strange +colonel +##mi +revolution +bell +politics +turns +silent +rail +relief +independence +combat +shape +write +determined +sales +learned +4th +finger +oxford +providing +1937 +heritage +fiction +situated +designated +allowing +distribution +hosted +##est +sight +interview +estimated +reduced +##ria +toronto +footballer +keeping +guys +damn +claim +motion +sport +sixth +stayed +##ze +en +rear +receive +handed +twelve +dress +audience +granted +brazil +##well +spirit +##ated +noticed +etc +olympic +representative +eric +tight +trouble +reviews +drink +vampire +missing +roles +ranked +newly +household +finals +wave +critics +##ee +phase +massachusetts +pilot +unlike +philadelphia +bright +guns +crown +organizations +roof +42 +respectively +clearly +tongue +marked +circle +fox +korea +bronze +brian +expanded +sexual +supply +yourself +inspired +labour +fc +##ah +reference +vision +draft +connection +brand +reasons +1935 +classic +driving +trip +jesus +cells +entry +1920 +neither +trail +claims +atlantic +orders +labor +nose +afraid +identified +intelligence +calls +cancer +attacked +passing +stephen +positions +imperial +grey +jason +39 +sunday +48 +swedish +avoid +extra +uncle +message +covers +allows +surprise +materials +fame +hunter +##ji +1930 +citizens +figures +davis +environmental +confirmed +shit +titles +di +performing +difference +acts +attacks +##ov +existing +votes +opportunity +nor +shop +entirely +trains +opposite +pakistan +##pa +develop +resulted +representatives +actions +reality +pressed +##ish +barely +wine +conversation +faculty +northwest +ends +documentary +nuclear +stock +grace +sets +eat +alternative +##ps +bag +resulting +creating +surprised +cemetery +1919 +drop +finding +sarah +cricket +streets +tradition +ride +1933 +exhibition +target +ear +explained +rain +composer +injury +apartment +municipal +educational +occupied +netherlands +clean +billion +constitution +learn +1914 +maximum +classical +francis +lose +opposition +jose +ontario +bear +core +hills +rolled +ending +drawn +permanent +fun +##tes +##lla +lewis +sites +chamber +ryan +##way +scoring +height +1934 +##house +lyrics +staring +55 +officials +1917 +snow +oldest +##tic +orange +##ger +qualified +interior +apparently +succeeded +thousand +dinner +lights +existence +fans +heavily +41 +greatest +conservative +send +bowl +plus +enter +catch +##un +economy +duty +1929 +speech +authorities +princess +performances +versions +shall +graduate +pictures +effective +remembered +poetry +desk +crossed +starring +starts +passenger +sharp +##ant +acres +ass +weather +falling +rank +fund +supporting +check +adult +publishing +heads +cm +southeast +lane +##burg +application +bc +##ura +les +condition +transfer +prevent +display +ex +regions +earl +federation +cool +relatively +answered +besides +1928 +obtained +portion +##town +mix +##ding +reaction +liked +dean +express +peak +1932 +##tte +counter +religion +chain +rare +miller +convention +aid +lie +vehicles +mobile +perform +squad +wonder +lying +crazy +sword +##ping +attempted +centuries +weren +philosophy +category +##ize +anna +interested +47 +sweden +wolf +frequently +abandoned +kg +literary +alliance +task +entitled +##ay +threw +promotion +factory +tiny +soccer +visited +matt +fm +achieved +52 +defence +internal +persian +43 +methods +##ging +arrested +otherwise +cambridge +programming +villages +elementary +districts +rooms +criminal +conflict +worry +trained +1931 +attempts +waited +signal +bird +truck +subsequent +programme +##ol +ad +49 +communist +details +faith +sector +patrick +carrying +laugh +##ss +controlled +korean +showing +origin +fuel +evil +1927 +##ent +brief +identity +darkness +address +pool +missed +publication +web +planet +ian +anne +wings +invited +##tt +briefly +standards +kissed +##be +ideas +climate +causing +walter +worse +albert +articles +winners +desire +aged +northeast +dangerous +gate +doubt +1922 +wooden +multi +##ky +poet +rising +funding +46 +communications +communication +violence +copies +prepared +ford +investigation +skills +1924 +pulling +electronic +##ak +##ial +##han +containing +ultimately +offices +singing +understanding +restaurant +tomorrow +fashion +christ +ward +da +pope +stands +5th +flow +studios +aired +commissioned +contained +exist +fresh +americans +##per +wrestling +approved +kid +employed +respect +suit +1925 +angel +asking +increasing +frame +angry +selling +1950s +thin +finds +##nd +temperature +statement +ali +explain +inhabitants +towns +extensive +narrow +51 +jane +flowers +images +promise +somewhere +object +fly +closely +##ls +1912 +bureau +cape +1926 +weekly +presidential +legislative +1921 +##ai +##au +launch +founding +##ny +978 +##ring +artillery +strike +un +institutions +roll +writers +landing +chose +kevin +anymore +pp +##ut +attorney +fit +dan +billboard +receiving +agricultural +breaking +sought +dave +admitted +lands +mexican +##bury +charlie +specifically +hole +iv +howard +credit +moscow +roads +accident +1923 +proved +wear +struck +hey +guards +stuff +slid +expansion +1915 +cat +anthony +##kin +melbourne +opposed +sub +southwest +architect +failure +plane +1916 +##ron +map +camera +tank +listen +regarding +wet +introduction +metropolitan +link +ep +fighter +inch +grown +gene +anger +fixed +buy +dvd +khan +domestic +worldwide +chapel +mill +functions +examples +##head +developing +1910 +turkey +hits +pocket +antonio +papers +grow +unless +circuit +18th +concerned +attached +journalist +selection +journey +converted +provincial +painted +hearing +aren +bands +negative +aside +wondered +knight +lap +survey +ma +##ow +noise +billy +##ium +shooting +guide +bedroom +priest +resistance +motor +homes +sounded +giant +##mer +150 +scenes +equal +comic +patients +hidden +solid +actual +bringing +afternoon +touched +funds +wedding +consisted +marie +canal +sr +kim +treaty +turkish +recognition +residence +cathedral +broad +knees +incident +shaped +fired +norwegian +handle +cheek +contest +represent +##pe +representing +beauty +##sen +birds +advantage +emergency +wrapped +drawing +notice +pink +broadcasting +##ong +somehow +bachelor +seventh +collected +registered +establishment +alan +assumed +chemical +personnel +roger +retirement +jeff +portuguese +wore +tied +device +threat +progress +advance +##ised +banks +hired +manchester +nfl +teachers +structures +forever +##bo +tennis +helping +saturday +sale +applications +junction +hip +incorporated +neighborhood +dressed +ceremony +##ds +influenced +hers +visual +stairs +decades +inner +kansas +hung +hoped +gain +scheduled +downtown +engaged +austria +clock +norway +certainly +pale +protected +1913 +victor +employees +plate +putting +surrounded +##ists +finishing +blues +tropical +##ries +minnesota +consider +philippines +accept +54 +retrieved +1900 +concern +anderson +properties +institution +gordon +successfully +vietnam +##dy +backing +outstanding +muslim +crossing +folk +producing +usual +demand +occurs +observed +lawyer +educated +##ana +kelly +string +pleasure +budget +items +quietly +colorado +philip +typical +##worth +derived +600 +survived +asks +mental +##ide +56 +jake +jews +distinguished +ltd +1911 +sri +extremely +53 +athletic +loud +thousands +worried +shadow +transportation +horses +weapon +arena +importance +users +tim +objects +contributed +dragon +douglas +aware +senator +johnny +jordan +sisters +engines +flag +investment +samuel +shock +capable +clark +row +wheel +refers +session +familiar +biggest +wins +hate +maintained +drove +hamilton +request +expressed +injured +underground +churches +walker +wars +tunnel +passes +stupid +agriculture +softly +cabinet +regarded +joining +indiana +##ea +##ms +push +dates +spend +behavior +woods +protein +gently +chase +morgan +mention +burning +wake +combination +occur +mirror +leads +jimmy +indeed +impossible +singapore +paintings +covering +##nes +soldier +locations +attendance +sell +historian +wisconsin +invasion +argued +painter +diego +changing +egypt +##don +experienced +inches +##ku +missouri +vol +grounds +spoken +switzerland +##gan +reform +rolling +ha +forget +massive +resigned +burned +allen +tennessee +locked +values +improved +##mo +wounded +universe +sick +dating +facing +pack +purchase +user +##pur +moments +##ul +merged +anniversary +1908 +coal +brick +understood +causes +dynasty +queensland +establish +stores +crisis +promote +hoping +views +cards +referee +extension +##si +raise +arizona +improve +colonial +formal +charged +##rt +palm +lucky +hide +rescue +faces +95 +feelings +candidates +juan +##ell +goods +6th +courses +weekend +59 +luke +cash +fallen +##om +delivered +affected +installed +carefully +tries +swiss +hollywood +costs +lincoln +responsibility +##he +shore +file +proper +normally +maryland +assistance +jump +constant +offering +friendly +waters +persons +realize +contain +trophy +800 +partnership +factor +58 +musicians +cry +bound +oregon +indicated +hero +houston +medium +##ure +consisting +somewhat +##ara +57 +cycle +##che +beer +moore +frederick +gotten +eleven +worst +weak +approached +arranged +chin +loan +universal +bond +fifteen +pattern +disappeared +##ney +translated +##zed +lip +arab +capture +interests +insurance +##chi +shifted +cave +prix +warning +sections +courts +coat +plot +smell +feed +golf +favorite +maintain +knife +vs +voted +degrees +finance +quebec +opinion +translation +manner +ruled +operate +productions +choose +musician +discovery +confused +tired +separated +stream +techniques +committed +attend +ranking +kings +throw +passengers +measure +horror +fan +mining +sand +danger +salt +calm +decade +dam +require +runner +##ik +rush +associate +greece +##ker +rivers +consecutive +matthew +##ski +sighed +sq +documents +steam +edited +closing +tie +accused +1905 +##ini +islamic +distributed +directors +organisation +bruce +7th +breathing +mad +lit +arrival +concrete +taste +08 +composition +shaking +faster +amateur +adjacent +stating +1906 +twin +flew +##ran +tokyo +publications +##tone +obviously +ridge +storage +1907 +carl +pages +concluded +desert +driven +universities +ages +terminal +sequence +borough +250 +constituency +creative +cousin +economics +dreams +margaret +notably +reduce +montreal +mode +17th +ears +saved +jan +vocal +##ica +1909 +andy +##jo +riding +roughly +threatened +##ise +meters +meanwhile +landed +compete +repeated +grass +czech +regularly +charges +tea +sudden +appeal +##ung +solution +describes +pierre +classification +glad +parking +##ning +belt +physics +99 +rachel +add +hungarian +participate +expedition +damaged +gift +childhood +85 +fifty +##red +mathematics +jumped +letting +defensive +mph +##ux +##gh +testing +##hip +hundreds +shoot +owners +matters +smoke +israeli +kentucky +dancing +mounted +grandfather +emma +designs +profit +argentina +##gs +truly +li +lawrence +cole +begun +detroit +willing +branches +smiling +decide +miami +enjoyed +recordings +##dale +poverty +ethnic +gay +##bi +gary +arabic +09 +accompanied +##one +##ons +fishing +determine +residential +acid +##ary +alice +returns +starred +mail +##ang +jonathan +strategy +##ue +net +forty +cook +businesses +equivalent +commonwealth +distinct +ill +##cy +seriously +##ors +##ped +shift +harris +replace +rio +imagine +formula +ensure +##ber +additionally +scheme +conservation +occasionally +purposes +feels +favor +##and +##ore +1930s +contrast +hanging +hunt +movies +1904 +instruments +victims +danish +christopher +busy +demon +sugar +earliest +colony +studying +balance +duties +##ks +belgium +slipped +carter +05 +visible +stages +iraq +fifa +##im +commune +forming +zero +07 +continuing +talked +counties +legend +bathroom +option +tail +clay +daughters +afterwards +severe +jaw +visitors +##ded +devices +aviation +russell +kate +##vi +entering +subjects +##ino +temporary +swimming +forth +smooth +ghost +audio +bush +operates +rocks +movements +signs +eddie +##tz +ann +voices +honorary +06 +memories +dallas +pure +measures +racial +promised +66 +harvard +ceo +16th +parliamentary +indicate +benefit +flesh +dublin +louisiana +1902 +1901 +patient +sleeping +1903 +membership +coastal +medieval +wanting +element +scholars +rice +62 +limit +survive +makeup +rating +definitely +collaboration +obvious +##tan +boss +ms +baron +birthday +linked +soil +diocese +##lan +ncaa +##mann +offensive +shell +shouldn +waist +##tus +plain +ross +organ +resolution +manufacturing +adding +relative +kennedy +98 +whilst +moth +marketing +gardens +crash +72 +heading +partners +credited +carlos +moves +cable +##zi +marshall +##out +depending +bottle +represents +rejected +responded +existed +04 +jobs +denmark +lock +##ating +treated +graham +routes +talent +commissioner +drugs +secure +tests +reign +restored +photography +##gi +contributions +oklahoma +designer +disc +grin +seattle +robin +paused +atlanta +unusual +##gate +praised +las +laughing +satellite +hungary +visiting +##sky +interesting +factors +deck +poems +norman +##water +stuck +speaker +rifle +domain +premiered +##her +dc +comics +actors +01 +reputation +eliminated +8th +ceiling +prisoners +script +##nce +leather +austin +mississippi +rapidly +admiral +parallel +charlotte +guilty +tools +gender +divisions +fruit +##bs +laboratory +nelson +fantasy +marry +rapid +aunt +tribe +requirements +aspects +suicide +amongst +adams +bone +ukraine +abc +kick +sees +edinburgh +clothing +column +rough +gods +hunting +broadway +gathered +concerns +##ek +spending +ty +12th +snapped +requires +solar +bones +cavalry +##tta +iowa +drinking +waste +index +franklin +charity +thompson +stewart +tip +flash +landscape +friday +enjoy +singh +poem +listening +##back +eighth +fred +differences +adapted +bomb +ukrainian +surgery +corporate +masters +anywhere +##more +waves +odd +sean +portugal +orleans +dick +debate +kent +eating +puerto +cleared +96 +expect +cinema +97 +guitarist +blocks +electrical +agree +involving +depth +dying +panel +struggle +##ged +peninsula +adults +novels +emerged +vienna +metro +debuted +shoes +tamil +songwriter +meets +prove +beating +instance +heaven +scared +sending +marks +artistic +passage +superior +03 +significantly +shopping +##tive +retained +##izing +malaysia +technique +cheeks +##ola +warren +maintenance +destroy +extreme +allied +120 +appearing +##yn +fill +advice +alabama +qualifying +policies +cleveland +hat +battery +smart +authors +10th +soundtrack +acted +dated +lb +glance +equipped +coalition +funny +outer +ambassador +roy +possibility +couples +campbell +dna +loose +ethan +supplies +1898 +gonna +88 +monster +##res +shake +agents +frequency +springs +dogs +practices +61 +gang +plastic +easier +suggests +gulf +blade +exposed +colors +industries +markets +pan +nervous +electoral +charts +legislation +ownership +##idae +mac +appointment +shield +copy +assault +socialist +abbey +monument +license +throne +employment +jay +93 +replacement +charter +cloud +powered +suffering +accounts +oak +connecticut +strongly +wright +colour +crystal +13th +context +welsh +networks +voiced +gabriel +jerry +##cing +forehead +mp +##ens +manage +schedule +totally +remix +##ii +forests +occupation +print +nicholas +brazilian +strategic +vampires +engineers +76 +roots +seek +correct +instrumental +und +alfred +backed +hop +##des +stanley +robinson +traveled +wayne +welcome +austrian +achieve +67 +exit +rates +1899 +strip +whereas +##cs +sing +deeply +adventure +bobby +rick +jamie +careful +components +cap +useful +personality +knee +##shi +pushing +hosts +02 +protest +ca +ottoman +symphony +##sis +63 +boundary +1890 +processes +considering +considerable +tons +##work +##ft +##nia +cooper +trading +dear +conduct +91 +illegal +apple +revolutionary +holiday +definition +harder +##van +jacob +circumstances +destruction +##lle +popularity +grip +classified +liverpool +donald +baltimore +flows +seeking +honour +approval +92 +mechanical +till +happening +statue +critic +increasingly +immediate +describe +commerce +stare +##ster +indonesia +meat +rounds +boats +baker +orthodox +depression +formally +worn +naked +claire +muttered +sentence +11th +emily +document +77 +criticism +wished +vessel +spiritual +bent +virgin +parker +minimum +murray +lunch +danny +printed +compilation +keyboards +false +blow +belonged +68 +raising +78 +cutting +##board +pittsburgh +##up +9th +shadows +81 +hated +indigenous +jon +15th +barry +scholar +ah +##zer +oliver +##gy +stick +susan +meetings +attracted +spell +romantic +##ver +ye +1895 +photo +demanded +customers +##ac +1896 +logan +revival +keys +modified +commanded +jeans +##ious +upset +raw +phil +detective +hiding +resident +vincent +##bly +experiences +diamond +defeating +coverage +lucas +external +parks +franchise +helen +bible +successor +percussion +celebrated +il +lift +profile +clan +romania +##ied +mills +##su +nobody +achievement +shrugged +fault +1897 +rhythm +initiative +breakfast +carbon +700 +69 +lasted +violent +74 +wound +ken +killer +gradually +filmed +°c +dollars +processing +94 +remove +criticized +guests +sang +chemistry +##vin +legislature +disney +##bridge +uniform +escaped +integrated +proposal +purple +denied +liquid +karl +influential +morris +nights +stones +intense +experimental +twisted +71 +84 +##ld +pace +nazi +mitchell +ny +blind +reporter +newspapers +14th +centers +burn +basin +forgotten +surviving +filed +collections +monastery +losses +manual +couch +description +appropriate +merely +tag +missions +sebastian +restoration +replacing +triple +73 +elder +julia +warriors +benjamin +julian +convinced +stronger +amazing +declined +versus +merchant +happens +output +finland +bare +barbara +absence +ignored +dawn +injuries +##port +producers +##ram +82 +luis +##ities +kw +admit +expensive +electricity +nba +exception +symbol +##ving +ladies +shower +sheriff +characteristics +##je +aimed +button +ratio +effectively +summit +angle +jury +bears +foster +vessels +pants +executed +evans +dozen +advertising +kicked +patrol +1889 +competitions +lifetime +principles +athletics +##logy +birmingham +sponsored +89 +rob +nomination +1893 +acoustic +##sm +creature +longest +##tra +credits +harbor +dust +josh +##so +territories +milk +infrastructure +completion +thailand +indians +leon +archbishop +##sy +assist +pitch +blake +arrangement +girlfriend +serbian +operational +hence +sad +scent +fur +dj +sessions +hp +refer +rarely +##ora +exists +1892 +##ten +scientists +dirty +penalty +burst +portrait +seed +79 +pole +limits +rival +1894 +stable +alpha +grave +constitutional +alcohol +arrest +flower +mystery +devil +architectural +relationships +greatly +habitat +##istic +larry +progressive +remote +cotton +##ics +##ok +preserved +reaches +##ming +cited +86 +vast +scholarship +decisions +cbs +joy +teach +1885 +editions +knocked +eve +searching +partly +participation +gap +animated +fate +excellent +##ett +na +87 +alternate +saints +youngest +##ily +climbed +##ita +##tors +suggest +##ct +discussion +staying +choir +lakes +jacket +revenue +nevertheless +peaked +instrument +wondering +annually +managing +neil +1891 +signing +terry +##ice +apply +clinical +brooklyn +aim +catherine +fuck +farmers +figured +ninth +pride +hugh +evolution +ordinary +involvement +comfortable +shouted +tech +encouraged +taiwan +representation +sharing +##lia +##em +panic +exact +cargo +competing +fat +cried +83 +1920s +occasions +pa +cabin +borders +utah +marcus +##isation +badly +muscles +##ance +victorian +transition +warner +bet +permission +##rin +slave +terrible +similarly +shares +seth +uefa +possession +medals +benefits +colleges +lowered +perfectly +mall +transit +##ye +##kar +publisher +##ened +harrison +deaths +elevation +##ae +asleep +machines +sigh +ash +hardly +argument +occasion +parent +leo +decline +1888 +contribution +##ua +concentration +1000 +opportunities +hispanic +guardian +extent +emotions +hips +mason +volumes +bloody +controversy +diameter +steady +mistake +phoenix +identify +violin +##sk +departure +richmond +spin +funeral +enemies +1864 +gear +literally +connor +random +sergeant +grab +confusion +1865 +transmission +informed +op +leaning +sacred +suspended +thinks +gates +portland +luck +agencies +yours +hull +expert +muscle +layer +practical +sculpture +jerusalem +latest +lloyd +statistics +deeper +recommended +warrior +arkansas +mess +supports +greg +eagle +1880 +recovered +rated +concerts +rushed +##ano +stops +eggs +files +premiere +keith +##vo +delhi +turner +pit +affair +belief +paint +##zing +mate +##ach +##ev +victim +##ology +withdrew +bonus +styles +fled +##ud +glasgow +technologies +funded +nbc +adaptation +##ata +portrayed +cooperation +supporters +judges +bernard +justin +hallway +ralph +##ick +graduating +controversial +distant +continental +spider +bite +##ho +recognize +intention +mixing +##ese +egyptian +bow +tourism +suppose +claiming +tiger +dominated +participants +vi +##ru +nurse +partially +tape +##rum +psychology +##rn +essential +touring +duo +voting +civilian +emotional +channels +##king +apparent +hebrew +1887 +tommy +carrier +intersection +beast +hudson +##gar +##zo +lab +nova +bench +discuss +costa +##ered +detailed +behalf +drivers +unfortunately +obtain +##lis +rocky +##dae +siege +friendship +honey +##rian +1861 +amy +hang +posted +governments +collins +respond +wildlife +preferred +operator +##po +laura +pregnant +videos +dennis +suspected +boots +instantly +weird +automatic +businessman +alleged +placing +throwing +ph +mood +1862 +perry +venue +jet +remainder +##lli +##ci +passion +biological +boyfriend +1863 +dirt +buffalo +ron +segment +fa +abuse +##era +genre +thrown +stroke +colored +stress +exercise +displayed +##gen +struggled +##tti +abroad +dramatic +wonderful +thereafter +madrid +component +widespread +##sed +tale +citizen +todd +monday +1886 +vancouver +overseas +forcing +crying +descent +##ris +discussed +substantial +ranks +regime +1870 +provinces +switch +drum +zane +ted +tribes +proof +lp +cream +researchers +volunteer +manor +silk +milan +donated +allies +venture +principle +delivery +enterprise +##ves +##ans +bars +traditionally +witch +reminded +copper +##uk +pete +inter +links +colin +grinned +elsewhere +competitive +frequent +##oy +scream +##hu +tension +texts +submarine +finnish +defending +defend +pat +detail +1884 +affiliated +stuart +themes +villa +periods +tool +belgian +ruling +crimes +answers +folded +licensed +resort +demolished +hans +lucy +1881 +lion +traded +photographs +writes +craig +##fa +trials +generated +beth +noble +debt +percentage +yorkshire +erected +ss +viewed +grades +confidence +ceased +islam +telephone +retail +##ible +chile +m² +roberts +sixteen +##ich +commented +hampshire +innocent +dual +pounds +checked +regulations +afghanistan +sung +rico +liberty +assets +bigger +options +angels +relegated +tribute +wells +attending +leaf +##yan +butler +romanian +forum +monthly +lisa +patterns +gmina +##tory +madison +hurricane +rev +##ians +bristol +##ula +elite +valuable +disaster +democracy +awareness +germans +freyja +##ins +loop +absolutely +paying +populations +maine +sole +prayer +spencer +releases +doorway +bull +##ani +lover +midnight +conclusion +##sson +thirteen +lily +mediterranean +##lt +nhl +proud +sample +##hill +drummer +guinea +##ova +murphy +climb +##ston +instant +attributed +horn +ain +railways +steven +##ao +autumn +ferry +opponent +root +traveling +secured +corridor +stretched +tales +sheet +trinity +cattle +helps +indicates +manhattan +murdered +fitted +1882 +gentle +grandmother +mines +shocked +vegas +produces +##light +caribbean +##ou +belong +continuous +desperate +drunk +historically +trio +waved +raf +dealing +nathan +bat +murmured +interrupted +residing +scientist +pioneer +harold +aaron +##net +delta +attempting +minority +mini +believes +chorus +tend +lots +eyed +indoor +load +shots +updated +jail +##llo +concerning +connecting +wealth +##ved +slaves +arrive +rangers +sufficient +rebuilt +##wick +cardinal +flood +muhammad +whenever +relation +runners +moral +repair +viewers +arriving +revenge +punk +assisted +bath +fairly +breathe +lists +innings +illustrated +whisper +nearest +voters +clinton +ties +ultimate +screamed +beijing +lions +andre +fictional +gathering +comfort +radar +suitable +dismissed +hms +ban +pine +wrist +atmosphere +voivodeship +bid +timber +##ned +##nan +giants +##ane +cameron +recovery +uss +identical +categories +switched +serbia +laughter +noah +ensemble +therapy +peoples +touching +##off +locally +pearl +platforms +everywhere +ballet +tables +lanka +herbert +outdoor +toured +derek +1883 +spaces +contested +swept +1878 +exclusive +slight +connections +##dra +winds +prisoner +collective +bangladesh +tube +publicly +wealthy +thai +##ys +isolated +select +##ric +insisted +pen +fortune +ticket +spotted +reportedly +animation +enforcement +tanks +110 +decides +wider +lowest +owen +##time +nod +hitting +##hn +gregory +furthermore +magazines +fighters +solutions +##ery +pointing +requested +peru +reed +chancellor +knights +mask +worker +eldest +flames +reduction +1860 +volunteers +##tis +reporting +##hl +wire +advisory +endemic +origins +settlers +pursue +knock +consumer +1876 +eu +compound +creatures +mansion +sentenced +ivan +deployed +guitars +frowned +involves +mechanism +kilometers +perspective +shops +maps +terminus +duncan +alien +fist +bridges +##pers +heroes +fed +derby +swallowed +##ros +patent +sara +illness +characterized +adventures +slide +hawaii +jurisdiction +##op +organised +##side +adelaide +walks +biology +se +##ties +rogers +swing +tightly +boundaries +##rie +prepare +implementation +stolen +##sha +certified +colombia +edwards +garage +##mm +recalled +##ball +rage +harm +nigeria +breast +##ren +furniture +pupils +settle +##lus +cuba +balls +client +alaska +21st +linear +thrust +celebration +latino +genetic +terror +##cia +##ening +lightning +fee +witness +lodge +establishing +skull +##ique +earning +hood +##ei +rebellion +wang +sporting +warned +missile +devoted +activist +porch +worship +fourteen +package +1871 +decorated +##shire +housed +##ock +chess +sailed +doctors +oscar +joan +treat +garcia +harbour +jeremy +##ire +traditions +dominant +jacques +##gon +##wan +relocated +1879 +amendment +sized +companion +simultaneously +volleyball +spun +acre +increases +stopping +loves +belongs +affect +drafted +tossed +scout +battles +1875 +filming +shoved +munich +tenure +vertical +romance +pc +##cher +argue +##ical +craft +ranging +www +opens +honest +tyler +yesterday +virtual +##let +muslims +reveal +snake +immigrants +radical +screaming +speakers +firing +saving +belonging +ease +lighting +prefecture +blame +farmer +hungry +grows +rubbed +beam +sur +subsidiary +##cha +armenian +sao +dropping +conventional +##fer +microsoft +reply +qualify +spots +1867 +sweat +festivals +##ken +immigration +physician +discover +exposure +sandy +explanation +isaac +implemented +##fish +hart +initiated +connect +stakes +presents +heights +householder +pleased +tourist +regardless +slip +closest +##ction +surely +sultan +brings +riley +preparation +aboard +slammed +baptist +experiment +ongoing +interstate +organic +playoffs +##ika +1877 +130 +##tar +hindu +error +tours +tier +plenty +arrangements +talks +trapped +excited +sank +ho +athens +1872 +denver +welfare +suburb +athletes +trick +diverse +belly +exclusively +yelled +1868 +##med +conversion +##ette +1874 +internationally +computers +conductor +abilities +sensitive +hello +dispute +measured +globe +rocket +prices +amsterdam +flights +tigers +inn +municipalities +emotion +references +3d +##mus +explains +airlines +manufactured +pm +archaeological +1873 +interpretation +devon +comment +##ites +settlements +kissing +absolute +improvement +suite +impressed +barcelona +sullivan +jefferson +towers +jesse +julie +##tin +##lu +grandson +hi +gauge +regard +rings +interviews +trace +raymond +thumb +departments +burns +serial +bulgarian +scores +demonstrated +##ix +1866 +kyle +alberta +underneath +romanized +##ward +relieved +acquisition +phrase +cliff +reveals +han +cuts +merger +custom +##dar +nee +gilbert +graduation +##nts +assessment +cafe +difficulty +demands +swung +democrat +jennifer +commons +1940s +grove +##yo +completing +focuses +sum +substitute +bearing +stretch +reception +##py +reflected +essentially +destination +pairs +##ched +survival +resource +##bach +promoting +doubles +messages +tear +##down +##fully +parade +florence +harvey +incumbent +partial +framework +900 +pedro +frozen +procedure +olivia +controls +##mic +shelter +personally +temperatures +##od +brisbane +tested +sits +marble +comprehensive +oxygen +leonard +##kov +inaugural +iranian +referring +quarters +attitude +##ivity +mainstream +lined +mars +dakota +norfolk +unsuccessful +##° +explosion +helicopter +congressional +##sing +inspector +bitch +seal +departed +divine +##ters +coaching +examination +punishment +manufacturer +sink +columns +unincorporated +signals +nevada +squeezed +dylan +dining +photos +martial +manuel +eighteen +elevator +brushed +plates +ministers +ivy +congregation +##len +slept +specialized +taxes +curve +restricted +negotiations +likes +statistical +arnold +inspiration +execution +bold +intermediate +significance +margin +ruler +wheels +gothic +intellectual +dependent +listened +eligible +buses +widow +syria +earn +cincinnati +collapsed +recipient +secrets +accessible +philippine +maritime +goddess +clerk +surrender +breaks +playoff +database +##ified +##lon +ideal +beetle +aspect +soap +regulation +strings +expand +anglo +shorter +crosses +retreat +tough +coins +wallace +directions +pressing +##oon +shipping +locomotives +comparison +topics +nephew +##mes +distinction +honors +travelled +sierra +ibn +##over +fortress +sa +recognised +carved +1869 +clients +##dan +intent +##mar +coaches +describing +bread +##ington +beaten +northwestern +##ona +merit +youtube +collapse +challenges +em +historians +objective +submitted +virus +attacking +drake +assume +##ere +diseases +marc +stem +leeds +##cus +##ab +farming +glasses +##lock +visits +nowhere +fellowship +relevant +carries +restaurants +experiments +101 +constantly +bases +targets +shah +tenth +opponents +verse +territorial +##ira +writings +corruption +##hs +instruction +inherited +reverse +emphasis +##vic +employee +arch +keeps +rabbi +watson +payment +uh +##ala +nancy +##tre +venice +fastest +sexy +banned +adrian +properly +ruth +touchdown +dollar +boards +metre +circles +edges +favour +comments +ok +travels +liberation +scattered +firmly +##ular +holland +permitted +diesel +kenya +den +originated +##ral +demons +resumed +dragged +rider +##rus +servant +blinked +extend +torn +##ias +##sey +input +meal +everybody +cylinder +kinds +camps +##fe +bullet +logic +##wn +croatian +evolved +healthy +fool +chocolate +wise +preserve +pradesh +##ess +respective +1850 +##ew +chicken +artificial +gross +corresponding +convicted +cage +caroline +dialogue +##dor +narrative +stranger +mario +br +christianity +failing +trent +commanding +buddhist +1848 +maurice +focusing +yale +bike +altitude +##ering +mouse +revised +##sley +veteran +##ig +pulls +theology +crashed +campaigns +legion +##ability +drag +excellence +customer +cancelled +intensity +excuse +##lar +liga +participating +contributing +printing +##burn +variable +##rk +curious +bin +legacy +renaissance +##my +symptoms +binding +vocalist +dancer +##nie +grammar +gospel +democrats +ya +enters +sc +diplomatic +hitler +##ser +clouds +mathematical +quit +defended +oriented +##heim +fundamental +hardware +impressive +equally +convince +confederate +guilt +chuck +sliding +##ware +magnetic +narrowed +petersburg +bulgaria +otto +phd +skill +##ama +reader +hopes +pitcher +reservoir +hearts +automatically +expecting +mysterious +bennett +extensively +imagined +seeds +monitor +fix +##ative +journalism +struggling +signature +ranch +encounter +photographer +observation +protests +##pin +influences +##hr +calendar +##all +cruz +croatia +locomotive +hughes +naturally +shakespeare +basement +hook +uncredited +faded +theories +approaches +dare +phillips +filling +fury +obama +##ain +efficient +arc +deliver +min +raid +breeding +inducted +leagues +efficiency +axis +montana +eagles +##ked +supplied +instructions +karen +picking +indicating +trap +anchor +practically +christians +tomb +vary +occasional +electronics +lords +readers +newcastle +faint +innovation +collect +situations +engagement +160 +claude +mixture +##feld +peer +tissue +logo +lean +##ration +°f +floors +##ven +architects +reducing +##our +##ments +rope +1859 +ottawa +##har +samples +banking +declaration +proteins +resignation +francois +saudi +advocate +exhibited +armor +twins +divorce +##ras +abraham +reviewed +jo +temporarily +matrix +physically +pulse +curled +##ena +difficulties +bengal +usage +##ban +annie +riders +certificate +##pi +holes +warsaw +distinctive +jessica +##mon +mutual +1857 +customs +circular +eugene +removal +loaded +mere +vulnerable +depicted +generations +dame +heir +enormous +lightly +climbing +pitched +lessons +pilots +nepal +ram +google +preparing +brad +louise +renowned +##₂ +liam +##ably +plaza +shaw +sophie +brilliant +bills +##bar +##nik +fucking +mainland +server +pleasant +seized +veterans +jerked +fail +beta +brush +radiation +stored +warmth +southeastern +nate +sin +raced +berkeley +joke +athlete +designation +trunk +##low +roland +qualification +archives +heels +artwork +receives +judicial +reserves +##bed +woke +installation +abu +floating +fake +lesser +excitement +interface +concentrated +addressed +characteristic +amanda +saxophone +monk +auto +##bus +releasing +egg +dies +interaction +defender +ce +outbreak +glory +loving +##bert +sequel +consciousness +http +awake +ski +enrolled +##ress +handling +rookie +brow +somebody +biography +warfare +amounts +contracts +presentation +fabric +dissolved +challenged +meter +psychological +lt +elevated +rally +accurate +##tha +hospitals +undergraduate +specialist +venezuela +exhibit +shed +nursing +protestant +fluid +structural +footage +jared +consistent +prey +##ska +succession +reflect +exile +lebanon +wiped +suspect +shanghai +resting +integration +preservation +marvel +variant +pirates +sheep +rounded +capita +sailing +colonies +manuscript +deemed +variations +clarke +functional +emerging +boxing +relaxed +curse +azerbaijan +heavyweight +nickname +editorial +rang +grid +tightened +earthquake +flashed +miguel +rushing +##ches +improvements +boxes +brooks +180 +consumption +molecular +felix +societies +repeatedly +variation +aids +civic +graphics +professionals +realm +autonomous +receiver +delayed +workshop +militia +chairs +trump +canyon +##point +harsh +extending +lovely +happiness +##jan +stake +eyebrows +embassy +wellington +hannah +##ella +sony +corners +bishops +swear +cloth +contents +xi +namely +commenced +1854 +stanford +nashville +courage +graphic +commitment +garrison +##bin +hamlet +clearing +rebels +attraction +literacy +cooking +ruins +temples +jenny +humanity +celebrate +hasn +freight +sixty +rebel +bastard +##art +newton +##ada +deer +##ges +##ching +smiles +delaware +singers +##ets +approaching +assists +flame +##ph +boulevard +barrel +planted +##ome +pursuit +##sia +consequences +posts +shallow +invitation +rode +depot +ernest +kane +rod +concepts +preston +topic +chambers +striking +blast +arrives +descendants +montgomery +ranges +worlds +##lay +##ari +span +chaos +praise +##ag +fewer +1855 +sanctuary +mud +fbi +##ions +programmes +maintaining +unity +harper +bore +handsome +closure +tournaments +thunder +nebraska +linda +facade +puts +satisfied +argentine +dale +cork +dome +panama +##yl +1858 +tasks +experts +##ates +feeding +equation +##las +##ida +##tu +engage +bryan +##ax +um +quartet +melody +disbanded +sheffield +blocked +gasped +delay +kisses +maggie +connects +##non +sts +poured +creator +publishers +##we +guided +ellis +extinct +hug +gaining +##ord +complicated +##bility +poll +clenched +investigate +##use +thereby +quantum +spine +cdp +humor +kills +administered +semifinals +##du +encountered +ignore +##bu +commentary +##maker +bother +roosevelt +140 +plains +halfway +flowing +cultures +crack +imprisoned +neighboring +airline +##ses +##view +##mate +##ec +gather +wolves +marathon +transformed +##ill +cruise +organisations +carol +punch +exhibitions +numbered +alarm +ratings +daddy +silently +##stein +queens +colours +impression +guidance +liu +tactical +##rat +marshal +della +arrow +##ings +rested +feared +tender +owns +bitter +advisor +escort +##ides +spare +farms +grants +##ene +dragons +encourage +colleagues +cameras +##und +sucked +pile +spirits +prague +statements +suspension +landmark +fence +torture +recreation +bags +permanently +survivors +pond +spy +predecessor +bombing +coup +##og +protecting +transformation +glow +##lands +##book +dug +priests +andrea +feat +barn +jumping +##chen +##ologist +##con +casualties +stern +auckland +pipe +serie +revealing +ba +##bel +trevor +mercy +spectrum +yang +consist +governing +collaborated +possessed +epic +comprises +blew +shane +##ack +lopez +honored +magical +sacrifice +judgment +perceived +hammer +mtv +baronet +tune +das +missionary +sheets +350 +neutral +oral +threatening +attractive +shade +aims +seminary +##master +estates +1856 +michel +wounds +refugees +manufacturers +##nic +mercury +syndrome +porter +##iya +##din +hamburg +identification +upstairs +purse +widened +pause +cared +breathed +affiliate +santiago +prevented +celtic +fisher +125 +recruited +byzantine +reconstruction +farther +##mp +diet +sake +au +spite +sensation +##ert +blank +separation +105 +##hon +vladimir +armies +anime +##lie +accommodate +orbit +cult +sofia +archive +##ify +##box +founders +sustained +disorder +honours +northeastern +mia +crops +violet +threats +blanket +fires +canton +followers +southwestern +prototype +voyage +assignment +altered +moderate +protocol +pistol +##eo +questioned +brass +lifting +1852 +math +authored +##ual +doug +dimensional +dynamic +##san +1851 +pronounced +grateful +quest +uncomfortable +boom +presidency +stevens +relating +politicians +chen +barrier +quinn +diana +mosque +tribal +cheese +palmer +portions +sometime +chester +treasure +wu +bend +download +millions +reforms +registration +##osa +consequently +monitoring +ate +preliminary +brandon +invented +ps +eaten +exterior +intervention +ports +documented +log +displays +lecture +sally +favourite +##itz +vermont +lo +invisible +isle +breed +##ator +journalists +relay +speaks +backward +explore +midfielder +actively +stefan +procedures +cannon +blond +kenneth +centered +servants +chains +libraries +malcolm +essex +henri +slavery +##hal +facts +fairy +coached +cassie +cats +washed +cop +##fi +announcement +item +2000s +vinyl +activated +marco +frontier +growled +curriculum +##das +loyal +accomplished +leslie +ritual +kenny +##00 +vii +napoleon +hollow +hybrid +jungle +stationed +friedrich +counted +##ulated +platinum +theatrical +seated +col +rubber +glen +1840 +diversity +healing +extends +id +provisions +administrator +columbus +##oe +tributary +te +assured +org +##uous +prestigious +examined +lectures +grammy +ronald +associations +bailey +allan +essays +flute +believing +consultant +proceedings +travelling +1853 +kit +kerala +yugoslavia +buddy +methodist +##ith +burial +centres +batman +##nda +discontinued +bo +dock +stockholm +lungs +severely +##nk +citing +manga +##ugh +steal +mumbai +iraqi +robot +celebrity +bride +broadcasts +abolished +pot +joel +overhead +franz +packed +reconnaissance +johann +acknowledged +introduce +handled +doctorate +developments +drinks +alley +palestine +##nis +##aki +proceeded +recover +bradley +grain +patch +afford +infection +nationalist +legendary +##ath +interchange +virtually +gen +gravity +exploration +amber +vital +wishes +powell +doctrine +elbow +screenplay +##bird +contribute +indonesian +pet +creates +##com +enzyme +kylie +discipline +drops +manila +hunger +##ien +layers +suffer +fever +bits +monica +keyboard +manages +##hood +searched +appeals +##bad +testament +grande +reid +##war +beliefs +congo +##ification +##dia +si +requiring +##via +casey +1849 +regret +streak +rape +depends +syrian +sprint +pound +tourists +upcoming +pub +##xi +tense +##els +practiced +echo +nationwide +guild +motorcycle +liz +##zar +chiefs +desired +elena +bye +precious +absorbed +relatives +booth +pianist +##mal +citizenship +exhausted +wilhelm +##ceae +##hed +noting +quarterback +urge +hectares +##gue +ace +holly +##tal +blonde +davies +parked +sustainable +stepping +twentieth +airfield +galaxy +nest +chip +##nell +tan +shaft +paulo +requirement +##zy +paradise +tobacco +trans +renewed +vietnamese +##cker +##ju +suggesting +catching +holmes +enjoying +md +trips +colt +holder +butterfly +nerve +reformed +cherry +bowling +trailer +carriage +goodbye +appreciate +toy +joshua +interactive +enabled +involve +##kan +collar +determination +bunch +facebook +recall +shorts +superintendent +episcopal +frustration +giovanni +nineteenth +laser +privately +array +circulation +##ovic +armstrong +deals +painful +permit +discrimination +##wi +aires +retiring +cottage +ni +##sta +horizon +ellen +jamaica +ripped +fernando +chapters +playstation +patron +lecturer +navigation +behaviour +genes +georgian +export +solomon +rivals +swift +seventeen +rodriguez +princeton +independently +sox +1847 +arguing +entity +casting +hank +criteria +oakland +geographic +milwaukee +reflection +expanding +conquest +dubbed +##tv +halt +brave +brunswick +doi +arched +curtis +divorced +predominantly +somerset +streams +ugly +zoo +horrible +curved +buenos +fierce +dictionary +vector +theological +unions +handful +stability +chan +punjab +segments +##lly +altar +ignoring +gesture +monsters +pastor +##stone +thighs +unexpected +operators +abruptly +coin +compiled +associates +improving +migration +pin +##ose +compact +collegiate +reserved +##urs +quarterfinals +roster +restore +assembled +hurry +oval +##cies +1846 +flags +martha +##del +victories +sharply +##rated +argues +deadly +neo +drawings +symbols +performer +##iel +griffin +restrictions +editing +andrews +java +journals +arabia +compositions +dee +pierce +removing +hindi +casino +runway +civilians +minds +nasa +hotels +##zation +refuge +rent +retain +potentially +conferences +suburban +conducting +##tto +##tions +##tle +descended +massacre +##cal +ammunition +terrain +fork +souls +counts +chelsea +durham +drives +cab +##bank +perth +realizing +palestinian +finn +simpson +##dal +betty +##ule +moreover +particles +cardinals +tent +evaluation +extraordinary +##oid +inscription +##works +wednesday +chloe +maintains +panels +ashley +trucks +##nation +cluster +sunlight +strikes +zhang +##wing +dialect +canon +##ap +tucked +##ws +collecting +##mas +##can +##sville +maker +quoted +evan +franco +aria +buying +cleaning +eva +closet +provision +apollo +clinic +rat +##ez +necessarily +ac +##gle +##ising +venues +flipped +cent +spreading +trustees +checking +authorized +##sco +disappointed +##ado +notion +duration +trumpet +hesitated +topped +brussels +rolls +theoretical +hint +define +aggressive +repeat +wash +peaceful +optical +width +allegedly +mcdonald +strict +copyright +##illa +investors +mar +jam +witnesses +sounding +miranda +michelle +privacy +hugo +harmony +##pp +valid +lynn +glared +nina +102 +headquartered +diving +boarding +gibson +##ncy +albanian +marsh +routine +dealt +enhanced +er +intelligent +substance +targeted +enlisted +discovers +spinning +observations +pissed +smoking +rebecca +capitol +visa +varied +costume +seemingly +indies +compensation +surgeon +thursday +arsenal +westminster +suburbs +rid +anglican +##ridge +knots +foods +alumni +lighter +fraser +whoever +portal +scandal +##ray +gavin +advised +instructor +flooding +terrorist +##ale +teenage +interim +senses +duck +teen +thesis +abby +eager +overcome +##ile +newport +glenn +rises +shame +##cc +prompted +priority +forgot +bomber +nicolas +protective +360 +cartoon +katherine +breeze +lonely +trusted +henderson +richardson +relax +banner +candy +palms +remarkable +##rio +legends +cricketer +essay +ordained +edmund +rifles +trigger +##uri +##away +sail +alert +1830 +audiences +penn +sussex +siblings +pursued +indianapolis +resist +rosa +consequence +succeed +avoided +1845 +##ulation +inland +##tie +##nna +counsel +profession +chronicle +hurried +##una +eyebrow +eventual +bleeding +innovative +cure +##dom +committees +accounting +con +scope +hardy +heather +tenor +gut +herald +codes +tore +scales +wagon +##oo +luxury +tin +prefer +fountain +triangle +bonds +darling +convoy +dried +traced +beings +troy +accidentally +slam +findings +smelled +joey +lawyers +outcome +steep +bosnia +configuration +shifting +toll +brook +performers +lobby +philosophical +construct +shrine +aggregate +boot +cox +phenomenon +savage +insane +solely +reynolds +lifestyle +##ima +nationally +holdings +consideration +enable +edgar +mo +mama +##tein +fights +relegation +chances +atomic +hub +conjunction +awkward +reactions +currency +finale +kumar +underwent +steering +elaborate +gifts +comprising +melissa +veins +reasonable +sunshine +chi +solve +trails +inhabited +elimination +ethics +huh +ana +molly +consent +apartments +layout +marines +##ces +hunters +bulk +##oma +hometown +##wall +##mont +cracked +reads +neighbouring +withdrawn +admission +wingspan +damned +anthology +lancashire +brands +batting +forgive +cuban +awful +##lyn +104 +dimensions +imagination +##ade +dante +##ship +tracking +desperately +goalkeeper +##yne +groaned +workshops +confident +burton +gerald +milton +circus +uncertain +slope +copenhagen +sophia +fog +philosopher +portraits +accent +cycling +varying +gripped +larvae +garrett +specified +scotia +mature +luther +kurt +rap +##kes +aerial +750 +ferdinand +heated +es +transported +##shan +safely +nonetheless +##orn +##gal +motors +demanding +##sburg +startled +##brook +ally +generate +caps +ghana +stained +demo +mentions +beds +ap +afterward +diary +##bling +utility +##iro +richards +1837 +conspiracy +conscious +shining +footsteps +observer +cyprus +urged +loyalty +developer +probability +olive +upgraded +gym +miracle +insects +graves +1844 +ourselves +hydrogen +amazon +katie +tickets +poets +##pm +planes +##pan +prevention +witnessed +dense +jin +randy +tang +warehouse +monroe +bang +archived +elderly +investigations +alec +granite +mineral +conflicts +controlling +aboriginal +carlo +##zu +mechanics +stan +stark +rhode +skirt +est +##berry +bombs +respected +##horn +imposed +limestone +deny +nominee +memphis +grabbing +disabled +##als +amusement +aa +frankfurt +corn +referendum +varies +slowed +disk +firms +unconscious +incredible +clue +sue +##zhou +twist +##cio +joins +idaho +chad +developers +computing +destroyer +103 +mortal +tucker +kingston +choices +yu +carson +1800 +os +whitney +geneva +pretend +dimension +staged +plateau +maya +##une +freestyle +##bc +rovers +hiv +##ids +tristan +classroom +prospect +##hus +honestly +diploma +lied +thermal +auxiliary +feast +unlikely +iata +##tel +morocco +pounding +treasury +lithuania +considerably +1841 +dish +1812 +geological +matching +stumbled +destroying +marched +brien +advances +cake +nicole +belle +settling +measuring +directing +##mie +tuesday +bassist +capabilities +stunned +fraud +torpedo +##list +##phone +anton +wisdom +surveillance +ruined +##ulate +lawsuit +healthcare +theorem +halls +trend +aka +horizontal +dozens +acquire +lasting +swim +hawk +gorgeous +fees +vicinity +decrease +adoption +tactics +##ography +pakistani +##ole +draws +##hall +willie +burke +heath +algorithm +integral +powder +elliott +brigadier +jackie +tate +varieties +darker +##cho +lately +cigarette +specimens +adds +##ree +##ensis +##inger +exploded +finalist +cia +murders +wilderness +arguments +nicknamed +acceptance +onwards +manufacture +robertson +jets +tampa +enterprises +blog +loudly +composers +nominations +1838 +ai +malta +inquiry +automobile +hosting +viii +rays +tilted +grief +museums +strategies +furious +euro +equality +cohen +poison +surrey +wireless +governed +ridiculous +moses +##esh +##room +vanished +##ito +barnes +attract +morrison +istanbul +##iness +absent +rotation +petition +janet +##logical +satisfaction +custody +deliberately +observatory +comedian +surfaces +pinyin +novelist +strictly +canterbury +oslo +monks +embrace +ibm +jealous +photograph +continent +dorothy +marina +doc +excess +holden +allegations +explaining +stack +avoiding +lance +storyline +majesty +poorly +spike +dos +bradford +raven +travis +classics +proven +voltage +pillow +fists +butt +1842 +interpreted +##car +1839 +gage +telegraph +lens +promising +expelled +casual +collector +zones +##min +silly +nintendo +##kh +##bra +downstairs +chef +suspicious +afl +flies +vacant +uganda +pregnancy +condemned +lutheran +estimates +cheap +decree +saxon +proximity +stripped +idiot +deposits +contrary +presenter +magnus +glacier +im +offense +edwin +##ori +upright +##long +bolt +##ois +toss +geographical +##izes +environments +delicate +marking +abstract +xavier +nails +windsor +plantation +occurring +equity +saskatchewan +fears +drifted +sequences +vegetation +revolt +##stic +1843 +sooner +fusion +opposing +nato +skating +1836 +secretly +ruin +lease +##oc +edit +##nne +flora +anxiety +ruby +##ological +##mia +tel +bout +taxi +emmy +frost +rainbow +compounds +foundations +rainfall +assassination +nightmare +dominican +##win +achievements +deserve +orlando +intact +armenia +##nte +calgary +valentine +106 +marion +proclaimed +theodore +bells +courtyard +thigh +gonzalez +console +troop +minimal +monte +everyday +##ence +##if +supporter +terrorism +buck +openly +presbyterian +activists +carpet +##iers +rubbing +uprising +##yi +cute +conceived +legally +##cht +millennium +cello +velocity +ji +rescued +cardiff +1835 +rex +concentrate +senators +beard +rendered +glowing +battalions +scouts +competitors +sculptor +catalogue +arctic +ion +raja +bicycle +wow +glancing +lawn +##woman +gentleman +lighthouse +publish +predicted +calculated +##val +variants +##gne +strain +##ui +winston +deceased +##nus +touchdowns +brady +caleb +sinking +echoed +crush +hon +blessed +protagonist +hayes +endangered +magnitude +editors +##tine +estimate +responsibilities +##mel +backup +laying +consumed +sealed +zurich +lovers +frustrated +##eau +ahmed +kicking +mit +treasurer +1832 +biblical +refuse +terrified +pump +agrees +genuine +imprisonment +refuses +plymouth +##hen +lou +##nen +tara +trembling +antarctic +ton +learns +##tas +crap +crucial +faction +atop +##borough +wrap +lancaster +odds +hopkins +erik +lyon +##eon +bros +##ode +snap +locality +tips +empress +crowned +cal +acclaimed +chuckled +##ory +clara +sends +mild +towel +##fl +##day +##а +wishing +assuming +interviewed +##bal +##die +interactions +eden +cups +helena +##lf +indie +beck +##fire +batteries +filipino +wizard +parted +##lam +traces +##born +rows +idol +albany +delegates +##ees +##sar +discussions +##ex +notre +instructed +belgrade +highways +suggestion +lauren +possess +orientation +alexandria +abdul +beats +salary +reunion +ludwig +alright +wagner +intimate +pockets +slovenia +hugged +brighton +merchants +cruel +stole +trek +slopes +repairs +enrollment +politically +underlying +promotional +counting +boeing +##bb +isabella +naming +##и +keen +bacteria +listing +separately +belfast +ussr +450 +lithuanian +anybody +ribs +sphere +martinez +cock +embarrassed +proposals +fragments +nationals +##fs +##wski +premises +fin +1500 +alpine +matched +freely +bounded +jace +sleeve +##af +gaming +pier +populated +evident +##like +frances +flooded +##dle +frightened +pour +trainer +framed +visitor +challenging +pig +wickets +##fold +infected +email +##pes +arose +##aw +reward +ecuador +oblast +vale +ch +shuttle +##usa +bach +rankings +forbidden +cornwall +accordance +salem +consumers +bruno +fantastic +toes +machinery +resolved +julius +remembering +propaganda +iceland +bombardment +tide +contacts +wives +##rah +concerto +macdonald +albania +implement +daisy +tapped +sudan +helmet +angela +mistress +##lic +crop +sunk +finest +##craft +hostile +##ute +##tsu +boxer +fr +paths +adjusted +habit +ballot +supervision +soprano +##zen +bullets +wicked +sunset +regiments +disappear +lamp +performs +app +##gia +##oa +rabbit +digging +incidents +entries +##cion +dishes +##oi +introducing +##ati +##fied +freshman +slot +jill +tackles +baroque +backs +##iest +lone +sponsor +destiny +altogether +convert +##aro +consensus +shapes +demonstration +basically +feminist +auction +artifacts +##bing +strongest +twitter +halifax +2019 +allmusic +mighty +smallest +precise +alexandra +viola +##los +##ille +manuscripts +##illo +dancers +ari +managers +monuments +blades +barracks +springfield +maiden +consolidated +electron +##end +berry +airing +wheat +nobel +inclusion +blair +payments +geography +bee +cc +eleanor +react +##hurst +afc +manitoba +##yu +su +lineup +fitness +recreational +investments +airborne +disappointment +##dis +edmonton +viewing +##row +renovation +##cast +infant +bankruptcy +roses +aftermath +pavilion +##yer +carpenter +withdrawal +ladder +##hy +discussing +popped +reliable +agreements +rochester +##abad +curves +bombers +220 +rao +reverend +decreased +choosing +107 +stiff +consulting +naples +crawford +tracy +ka +ribbon +cops +##lee +crushed +deciding +unified +teenager +accepting +flagship +explorer +poles +sanchez +inspection +revived +skilled +induced +exchanged +flee +locals +tragedy +swallow +loading +hanna +demonstrate +##ela +salvador +flown +contestants +civilization +##ines +wanna +rhodes +fletcher +hector +knocking +considers +##ough +nash +mechanisms +sensed +mentally +walt +unclear +##eus +renovated +madame +##cks +crews +governmental +##hin +undertaken +monkey +##ben +##ato +fatal +armored +copa +caves +governance +grasp +perception +certification +froze +damp +tugged +wyoming +##rg +##ero +newman +##lor +nerves +curiosity +graph +115 +##ami +withdraw +tunnels +dull +meredith +moss +exhibits +neighbors +communicate +accuracy +explored +raiders +republicans +secular +kat +superman +penny +criticised +##tch +freed +update +conviction +wade +ham +likewise +delegation +gotta +doll +promises +technological +myth +nationality +resolve +convent +##mark +sharon +dig +sip +coordinator +entrepreneur +fold +##dine +capability +councillor +synonym +blown +swan +cursed +1815 +jonas +haired +sofa +canvas +keeper +rivalry +##hart +rapper +speedway +swords +postal +maxwell +estonia +potter +recurring +##nn +##ave +errors +##oni +cognitive +1834 +##² +claws +nadu +roberto +bce +wrestler +ellie +##ations +infinite +ink +##tia +presumably +finite +staircase +108 +noel +patricia +nacional +##cation +chill +eternal +tu +preventing +prussia +fossil +limbs +##logist +ernst +frog +perez +rene +##ace +pizza +prussian +##ios +##vy +molecules +regulatory +answering +opinions +sworn +lengths +supposedly +hypothesis +upward +habitats +seating +ancestors +drank +yield +hd +synthesis +researcher +modest +##var +mothers +peered +voluntary +homeland +##the +acclaim +##igan +static +valve +luxembourg +alto +carroll +fe +receptor +norton +ambulance +##tian +johnston +catholics +depicting +jointly +elephant +gloria +mentor +badge +ahmad +distinguish +remarked +councils +precisely +allison +advancing +detection +crowded +##10 +cooperative +ankle +mercedes +dagger +surrendered +pollution +commit +subway +jeffrey +lesson +sculptures +provider +##fication +membrane +timothy +rectangular +fiscal +heating +teammate +basket +particle +anonymous +deployment +##ple +missiles +courthouse +proportion +shoe +sec +##ller +complaints +forbes +blacks +abandon +remind +sizes +overwhelming +autobiography +natalie +##awa +risks +contestant +countryside +babies +scorer +invaded +enclosed +proceed +hurling +disorders +##cu +reflecting +continuously +cruiser +graduates +freeway +investigated +ore +deserved +maid +blocking +phillip +jorge +shakes +dove +mann +variables +lacked +burden +accompanying +que +consistently +organizing +provisional +complained +endless +##rm +tubes +juice +georges +krishna +mick +labels +thriller +##uch +laps +arcade +sage +snail +##table +shannon +fi +laurence +seoul +vacation +presenting +hire +churchill +surprisingly +prohibited +savannah +technically +##oli +170 +##lessly +testimony +suited +speeds +toys +romans +mlb +flowering +measurement +talented +kay +settings +charleston +expectations +shattered +achieving +triumph +ceremonies +portsmouth +lanes +mandatory +loser +stretching +cologne +realizes +seventy +cornell +careers +webb +##ulating +americas +budapest +ava +suspicion +##ison +yo +conrad +##hai +sterling +jessie +rector +##az +1831 +transform +organize +loans +christine +volcanic +warrant +slender +summers +subfamily +newer +danced +dynamics +rhine +proceeds +heinrich +gastropod +commands +sings +facilitate +easter +ra +positioned +responses +expense +fruits +yanked +imported +25th +velvet +vic +primitive +tribune +baldwin +neighbourhood +donna +rip +hay +pr +##uro +1814 +espn +welcomed +##aria +qualifier +glare +highland +timing +##cted +shells +eased +geometry +louder +exciting +slovakia +##sion +##iz +##lot +savings +prairie +##ques +marching +rafael +tonnes +##lled +curtain +preceding +shy +heal +greene +worthy +##pot +detachment +bury +sherman +##eck +reinforced +seeks +bottles +contracted +duchess +outfit +walsh +##sc +mickey +##ase +geoffrey +archer +squeeze +dawson +eliminate +invention +##enberg +neal +##eth +stance +dealer +coral +maple +retire +polo +simplified +##ht +1833 +hid +watts +backwards +jules +##oke +genesis +mt +frames +rebounds +burma +woodland +moist +santos +whispers +drained +subspecies +##aa +streaming +ulster +burnt +correspondence +maternal +gerard +denis +stealing +##load +genius +duchy +##oria +inaugurated +momentum +suits +placement +sovereign +clause +thames +##hara +confederation +reservation +sketch +yankees +lets +rotten +charm +hal +verses +ultra +commercially +dot +salon +citation +adopt +winnipeg +mist +allocated +cairo +##boy +jenkins +interference +objectives +##wind +1820 +portfolio +armoured +sectors +##eh +initiatives +##world +integrity +exercises +robe +tap +ab +gazed +##tones +distracted +rulers +111 +favorable +jerome +tended +cart +factories +##eri +diplomat +valued +gravel +charitable +##try +calvin +exploring +chang +shepherd +terrace +pdf +pupil +##ural +reflects +ups +##rch +governors +shelf +depths +##nberg +trailed +crest +tackle +##nian +##ats +hatred +##kai +clare +makers +ethiopia +longtime +detected +embedded +lacking +slapped +rely +thomson +anticipation +iso +morton +successive +agnes +screenwriter +straightened +philippe +playwright +haunted +licence +iris +intentions +sutton +112 +logical +correctly +##weight +branded +licked +tipped +silva +ricky +narrator +requests +##ents +greeted +supernatural +cow +##wald +lung +refusing +employer +strait +gaelic +liner +##piece +zoe +sabha +##mba +driveway +harvest +prints +bates +reluctantly +threshold +algebra +ira +wherever +coupled +240 +assumption +picks +##air +designers +raids +gentlemen +##ean +roller +blowing +leipzig +locks +screw +dressing +strand +##lings +scar +dwarf +depicts +##nu +nods +##mine +differ +boris +##eur +yuan +flip +##gie +mob +invested +questioning +applying +##ture +shout +##sel +gameplay +blamed +illustrations +bothered +weakness +rehabilitation +##of +##zes +envelope +rumors +miners +leicester +subtle +kerry +##ico +ferguson +##fu +premiership +ne +##cat +bengali +prof +catches +remnants +dana +##rily +shouting +presidents +baltic +ought +ghosts +dances +sailors +shirley +fancy +dominic +##bie +madonna +##rick +bark +buttons +gymnasium +ashes +liver +toby +oath +providence +doyle +evangelical +nixon +cement +carnegie +embarked +hatch +surroundings +guarantee +needing +pirate +essence +##bee +filter +crane +hammond +projected +immune +percy +twelfth +##ult +regent +doctoral +damon +mikhail +##ichi +lu +critically +elect +realised +abortion +acute +screening +mythology +steadily +##fc +frown +nottingham +kirk +wa +minneapolis +##rra +module +algeria +mc +nautical +encounters +surprising +statues +availability +shirts +pie +alma +brows +munster +mack +soup +crater +tornado +sanskrit +cedar +explosive +bordered +dixon +planets +stamp +exam +happily +##bble +carriers +kidnapped +##vis +accommodation +emigrated +##met +knockout +correspondent +violation +profits +peaks +lang +specimen +agenda +ancestry +pottery +spelling +equations +obtaining +ki +linking +1825 +debris +asylum +##20 +buddhism +teddy +##ants +gazette +##nger +##sse +dental +eligibility +utc +fathers +averaged +zimbabwe +francesco +coloured +hissed +translator +lynch +mandate +humanities +mackenzie +uniforms +lin +##iana +##gio +asset +mhz +fitting +samantha +genera +wei +rim +beloved +shark +riot +entities +expressions +indo +carmen +slipping +owing +abbot +neighbor +sidney +##av +rats +recommendations +encouraging +squadrons +anticipated +commanders +conquered +##oto +donations +diagnosed +##mond +divide +##iva +guessed +decoration +vernon +auditorium +revelation +conversations +##kers +##power +herzegovina +dash +alike +protested +lateral +herman +accredited +mg +##gent +freeman +mel +fiji +crow +crimson +##rine +livestock +##pped +humanitarian +bored +oz +whip +##lene +##ali +legitimate +alter +grinning +spelled +anxious +oriental +wesley +##nin +##hole +carnival +controller +detect +##ssa +bowed +educator +kosovo +macedonia +##sin +occupy +mastering +stephanie +janeiro +para +unaware +nurses +noon +135 +cam +hopefully +ranger +combine +sociology +polar +rica +##eer +neill +##sman +holocaust +##ip +doubled +lust +1828 +109 +decent +cooling +unveiled +##card +1829 +nsw +homer +chapman +meyer +##gin +dive +mae +reagan +expertise +##gled +darwin +brooke +sided +prosecution +investigating +comprised +petroleum +genres +reluctant +differently +trilogy +johns +vegetables +corpse +highlighted +lounge +pension +unsuccessfully +elegant +aided +ivory +beatles +amelia +cain +dubai +sunny +immigrant +babe +click +##nder +underwater +pepper +combining +mumbled +atlas +horns +accessed +ballad +physicians +homeless +gestured +rpm +freak +louisville +corporations +patriots +prizes +rational +warn +modes +decorative +overnight +din +troubled +phantom +##ort +monarch +sheer +##dorf +generals +guidelines +organs +addresses +##zon +enhance +curling +parishes +cord +##kie +linux +caesar +deutsche +bavaria +##bia +coleman +cyclone +##eria +bacon +petty +##yama +##old +hampton +diagnosis +1824 +throws +complexity +rita +disputed +##₃ +pablo +##sch +marketed +trafficking +##ulus +examine +plague +formats +##oh +vault +faithful +##bourne +webster +##ox +highlights +##ient +##ann +phones +vacuum +sandwich +modeling +##gated +bolivia +clergy +qualities +isabel +##nas +##ars +wears +screams +reunited +annoyed +bra +##ancy +##rate +differential +transmitter +tattoo +container +poker +##och +excessive +resides +cowboys +##tum +augustus +trash +providers +statute +retreated +balcony +reversed +void +storey +preceded +masses +leap +laughs +neighborhoods +wards +schemes +falcon +santo +battlefield +pad +ronnie +thread +lesbian +venus +##dian +beg +sandstone +daylight +punched +gwen +analog +stroked +wwe +acceptable +measurements +dec +toxic +##kel +adequate +surgical +economist +parameters +varsity +##sberg +quantity +ella +##chy +##rton +countess +generating +precision +diamonds +expressway +ga +##ı +1821 +uruguay +talents +galleries +expenses +scanned +colleague +outlets +ryder +lucien +##ila +paramount +##bon +syracuse +dim +fangs +gown +sweep +##sie +toyota +missionaries +websites +##nsis +sentences +adviser +val +trademark +spells +##plane +patience +starter +slim +##borg +toe +incredibly +shoots +elliot +nobility +##wyn +cowboy +endorsed +gardner +tendency +persuaded +organisms +emissions +kazakhstan +amused +boring +chips +themed +##hand +llc +constantinople +chasing +systematic +guatemala +borrowed +erin +carey +##hard +highlands +struggles +1810 +##ifying +##ced +wong +exceptions +develops +enlarged +kindergarten +castro +##ern +##rina +leigh +zombie +juvenile +##most +consul +##nar +sailor +hyde +clarence +intensive +pinned +nasty +useless +jung +clayton +stuffed +exceptional +ix +apostolic +230 +transactions +##dge +exempt +swinging +cove +religions +##ash +shields +dairy +bypass +190 +pursuing +bug +joyce +bombay +chassis +southampton +chat +interact +redesignated +##pen +nascar +pray +salmon +rigid +regained +malaysian +grim +publicity +constituted +capturing +toilet +delegate +purely +tray +drift +loosely +striker +weakened +trinidad +mitch +itv +defines +transmitted +ming +scarlet +nodding +fitzgerald +fu +narrowly +sp +tooth +standings +virtue +##₁ +##wara +##cting +chateau +gloves +lid +##nel +hurting +conservatory +##pel +sinclair +reopened +sympathy +nigerian +strode +advocated +optional +chronic +discharge +##rc +suck +compatible +laurel +stella +shi +fails +wage +dodge +128 +informal +sorts +levi +buddha +villagers +##aka +chronicles +heavier +summoned +gateway +3000 +eleventh +jewelry +translations +accordingly +seas +##ency +fiber +pyramid +cubic +dragging +##ista +caring +##ops +android +contacted +lunar +##dt +kai +lisbon +patted +1826 +sacramento +theft +madagascar +subtropical +disputes +ta +holidays +piper +willow +mare +cane +itunes +newfoundland +benny +companions +dong +raj +observe +roar +charming +plaque +tibetan +fossils +enacted +manning +bubble +tina +tanzania +##eda +##hir +funk +swamp +deputies +cloak +ufc +scenario +par +scratch +metals +anthem +guru +engaging +specially +##boat +dialects +nineteen +cecil +duet +disability +messenger +unofficial +##lies +defunct +eds +moonlight +drainage +surname +puzzle +honda +switching +conservatives +mammals +knox +broadcaster +sidewalk +cope +##ried +benson +princes +peterson +##sal +bedford +sharks +eli +wreck +alberto +gasp +archaeology +lgbt +teaches +securities +madness +compromise +waving +coordination +davidson +visions +leased +possibilities +eighty +jun +fernandez +enthusiasm +assassin +sponsorship +reviewer +kingdoms +estonian +laboratories +##fy +##nal +applies +verb +celebrations +##zzo +rowing +lightweight +sadness +submit +mvp +balanced +dude +##vas +explicitly +metric +magnificent +mound +brett +mohammad +mistakes +irregular +##hing +##ass +sanders +betrayed +shipped +surge +##enburg +reporters +termed +georg +pity +verbal +bulls +abbreviated +enabling +appealed +##are +##atic +sicily +sting +heel +sweetheart +bart +spacecraft +brutal +monarchy +##tter +aberdeen +cameo +diane +##ub +survivor +clyde +##aries +complaint +##makers +clarinet +delicious +chilean +karnataka +coordinates +1818 +panties +##rst +pretending +ar +dramatically +kiev +bella +tends +distances +113 +catalog +launching +instances +telecommunications +portable +lindsay +vatican +##eim +angles +aliens +marker +stint +screens +bolton +##rne +judy +wool +benedict +plasma +europa +spark +imaging +filmmaker +swiftly +##een +contributor +##nor +opted +stamps +apologize +financing +butter +gideon +sophisticated +alignment +avery +chemicals +yearly +speculation +prominence +professionally +##ils +immortal +institutional +inception +wrists +identifying +tribunal +derives +gains +##wo +papal +preference +linguistic +vince +operative +brewery +##ont +unemployment +boyd +##ured +##outs +albeit +prophet +1813 +bi +##rr +##face +##rad +quarterly +asteroid +cleaned +radius +temper +##llen +telugu +jerk +viscount +menu +##ote +glimpse +##aya +yacht +hawaiian +baden +##rl +laptop +readily +##gu +monetary +offshore +scots +watches +##yang +##arian +upgrade +needle +xbox +lea +encyclopedia +flank +fingertips +##pus +delight +teachings +confirm +roth +beaches +midway +winters +##iah +teasing +daytime +beverly +gambling +bonnie +##backs +regulated +clement +hermann +tricks +knot +##shing +##uring +##vre +detached +ecological +owed +specialty +byron +inventor +bats +stays +screened +unesco +midland +trim +affection +##ander +##rry +jess +thoroughly +feedback +##uma +chennai +strained +heartbeat +wrapping +overtime +pleaded +##sworth +mon +leisure +oclc +##tate +##ele +feathers +angelo +thirds +nuts +surveys +clever +gill +commentator +##dos +darren +rides +gibraltar +##nc +##mu +dissolution +dedication +shin +meals +saddle +elvis +reds +chaired +taller +appreciation +functioning +niece +favored +advocacy +robbie +criminals +suffolk +yugoslav +passport +constable +congressman +hastings +vera +##rov +consecrated +sparks +ecclesiastical +confined +##ovich +muller +floyd +nora +1822 +paved +1827 +cumberland +ned +saga +spiral +##flow +appreciated +yi +collaborative +treating +similarities +feminine +finishes +##ib +jade +import +##nse +##hot +champagne +mice +securing +celebrities +helsinki +attributes +##gos +cousins +phases +ache +lucia +gandhi +submission +vicar +spear +shine +tasmania +biting +detention +constitute +tighter +seasonal +##gus +terrestrial +matthews +##oka +effectiveness +parody +philharmonic +##onic +1816 +strangers +encoded +consortium +guaranteed +regards +shifts +tortured +collision +supervisor +inform +broader +insight +theaters +armour +emeritus +blink +incorporates +mapping +##50 +##ein +handball +flexible +##nta +substantially +generous +thief +##own +carr +loses +1793 +prose +ucla +romeo +generic +metallic +realization +damages +mk +commissioners +zach +default +##ther +helicopters +lengthy +stems +spa +partnered +spectators +rogue +indication +penalties +teresa +1801 +sen +##tric +dalton +##wich +irving +photographic +##vey +dell +deaf +peters +excluded +unsure +##vable +patterson +crawled +##zio +resided +whipped +latvia +slower +ecole +pipes +employers +maharashtra +comparable +va +textile +pageant +##gel +alphabet +binary +irrigation +chartered +choked +antoine +offs +waking +supplement +##wen +quantities +demolition +regain +locate +urdu +folks +alt +114 +##mc +scary +andreas +whites +##ava +classrooms +mw +aesthetic +publishes +valleys +guides +cubs +johannes +bryant +conventions +affecting +##itt +drain +awesome +isolation +prosecutor +ambitious +apology +captive +downs +atmospheric +lorenzo +aisle +beef +foul +##onia +kidding +composite +disturbed +illusion +natives +##ffer +emi +rockets +riverside +wartime +painters +adolf +melted +##ail +uncertainty +simulation +hawks +progressed +meantime +builder +spray +breach +unhappy +regina +russians +##urg +determining +##tation +tram +1806 +##quin +aging +##12 +1823 +garion +rented +mister +diaz +terminated +clip +1817 +depend +nervously +disco +owe +defenders +shiva +notorious +disbelief +shiny +worcester +##gation +##yr +trailing +undertook +islander +belarus +limitations +watershed +fuller +overlooking +utilized +raphael +1819 +synthetic +breakdown +klein +##nate +moaned +memoir +lamb +practicing +##erly +cellular +arrows +exotic +##graphy +witches +117 +charted +rey +hut +hierarchy +subdivision +freshwater +giuseppe +aloud +reyes +qatar +marty +sideways +utterly +sexually +jude +prayers +mccarthy +softball +blend +damien +##gging +##metric +wholly +erupted +lebanese +negro +revenues +tasted +comparative +teamed +transaction +labeled +maori +sovereignty +parkway +trauma +gran +malay +121 +advancement +descendant +2020 +buzz +salvation +inventory +symbolic +##making +antarctica +mps +##gas +##bro +mohammed +myanmar +holt +submarines +tones +##lman +locker +patriarch +bangkok +emerson +remarks +predators +kin +afghan +confession +norwich +rental +emerge +advantages +##zel +rca +##hold +shortened +storms +aidan +##matic +autonomy +compliance +##quet +dudley +atp +##osis +1803 +motto +documentation +summary +professors +spectacular +christina +archdiocese +flashing +innocence +remake +##dell +psychic +reef +scare +employ +rs +sticks +meg +gus +leans +##ude +accompany +bergen +tomas +##iko +doom +wages +pools +##nch +##bes +breasts +scholarly +alison +outline +brittany +breakthrough +willis +realistic +##cut +##boro +competitor +##stan +pike +picnic +icon +designing +commercials +washing +villain +skiing +micro +costumes +auburn +halted +executives +##hat +logistics +cycles +vowel +applicable +barrett +exclaimed +eurovision +eternity +ramon +##umi +##lls +modifications +sweeping +disgust +##uck +torch +aviv +ensuring +rude +dusty +sonic +donovan +outskirts +cu +pathway +##band +##gun +##lines +disciplines +acids +cadet +paired +##40 +sketches +##sive +marriages +##⁺ +folding +peers +slovak +implies +admired +##beck +1880s +leopold +instinct +attained +weston +megan +horace +##ination +dorsal +ingredients +evolutionary +##its +complications +deity +lethal +brushing +levy +deserted +institutes +posthumously +delivering +telescope +coronation +motivated +rapids +luc +flicked +pays +volcano +tanner +weighed +##nica +crowds +frankie +gifted +addressing +granddaughter +winding +##rna +constantine +gomez +##front +landscapes +rudolf +anthropology +slate +werewolf +##lio +astronomy +circa +rouge +dreaming +sack +knelt +drowned +naomi +prolific +tracked +freezing +herb +##dium +agony +randall +twisting +wendy +deposit +touches +vein +wheeler +##bbled +##bor +batted +retaining +tire +presently +compare +specification +daemon +nigel +##grave +merry +recommendation +czechoslovakia +sandra +ng +roma +##sts +lambert +inheritance +sheikh +winchester +cries +examining +##yle +comeback +cuisine +nave +##iv +ko +retrieve +tomatoes +barker +polished +defining +irene +lantern +personalities +begging +tract +swore +1809 +175 +##gic +omaha +brotherhood +##rley +haiti +##ots +exeter +##ete +##zia +steele +dumb +pearson +210 +surveyed +elisabeth +trends +##ef +fritz +##rf +premium +bugs +fraction +calmly +viking +##birds +tug +inserted +unusually +##ield +confronted +distress +crashing +brent +turks +resign +##olo +cambodia +gabe +sauce +##kal +evelyn +116 +extant +clusters +quarry +teenagers +luna +##lers +##ister +affiliation +drill +##ashi +panthers +scenic +libya +anita +strengthen +inscriptions +##cated +lace +sued +judith +riots +##uted +mint +##eta +preparations +midst +dub +challenger +##vich +mock +cf +displaced +wicket +breaths +enables +schmidt +analyst +##lum +ag +highlight +automotive +axe +josef +newark +sufficiently +resembles +50th +##pal +flushed +mum +traits +##ante +commodore +incomplete +warming +titular +ceremonial +ethical +118 +celebrating +eighteenth +cao +lima +medalist +mobility +strips +snakes +##city +miniature +zagreb +barton +escapes +umbrella +automated +doubted +differs +cooled +georgetown +dresden +cooked +fade +wyatt +rna +jacobs +carlton +abundant +stereo +boost +madras +inning +##hia +spur +ip +malayalam +begged +osaka +groan +escaping +charging +dose +vista +##aj +bud +papa +communists +advocates +edged +tri +##cent +resemble +peaking +necklace +fried +montenegro +saxony +goose +glances +stuttgart +curator +recruit +grocery +sympathetic +##tting +##fort +127 +lotus +randolph +ancestor +##rand +succeeding +jupiter +1798 +macedonian +##heads +hiking +1808 +handing +fischer +##itive +garbage +node +##pies +prone +singular +papua +inclined +attractions +italia +pouring +motioned +grandma +garnered +jacksonville +corp +ego +ringing +aluminum +##hausen +ordering +##foot +drawer +traders +synagogue +##play +##kawa +resistant +wandering +fragile +fiona +teased +var +hardcore +soaked +jubilee +decisive +exposition +mercer +poster +valencia +hale +kuwait +1811 +##ises +##wr +##eed +tavern +gamma +122 +johan +##uer +airways +amino +gil +##ury +vocational +domains +torres +##sp +generator +folklore +outcomes +##keeper +canberra +shooter +fl +beams +confrontation +##lling +##gram +feb +aligned +forestry +pipeline +jax +motorway +conception +decay +##tos +coffin +##cott +stalin +1805 +escorted +minded +##nam +sitcom +purchasing +twilight +veronica +additions +passive +tensions +straw +123 +frequencies +1804 +refugee +cultivation +##iate +christie +clary +bulletin +crept +disposal +##rich +##zong +processor +crescent +##rol +bmw +emphasized +whale +nazis +aurora +##eng +dwelling +hauled +sponsors +toledo +mega +ideology +theatres +tessa +cerambycidae +saves +turtle +cone +suspects +kara +rusty +yelling +greeks +mozart +shades +cocked +participant +##tro +shire +spit +freeze +necessity +##cos +inmates +nielsen +councillors +loaned +uncommon +omar +peasants +botanical +offspring +daniels +formations +jokes +1794 +pioneers +sigma +licensing +##sus +wheelchair +polite +1807 +liquor +pratt +trustee +##uta +forewings +balloon +##zz +kilometre +camping +explicit +casually +shawn +foolish +teammates +nm +hassan +carrie +judged +satisfy +vanessa +knives +selective +cnn +flowed +##lice +eclipse +stressed +eliza +mathematician +cease +cultivated +##roy +commissions +browns +##ania +destroyers +sheridan +meadow +##rius +minerals +##cial +downstream +clash +gram +memoirs +ventures +baha +seymour +archie +midlands +edith +fare +flynn +invite +canceled +tiles +stabbed +boulder +incorporate +amended +camden +facial +mollusk +unreleased +descriptions +yoga +grabs +550 +raises +ramp +shiver +##rose +coined +pioneering +tunes +qing +warwick +tops +119 +melanie +giles +##rous +wandered +##inal +annexed +nov +30th +unnamed +##ished +organizational +airplane +normandy +stoke +whistle +blessing +violations +chased +holders +shotgun +##ctic +outlet +reactor +##vik +tires +tearing +shores +fortified +mascot +constituencies +nc +columnist +productive +tibet +##rta +lineage +hooked +oct +tapes +judging +cody +##gger +hansen +kashmir +triggered +##eva +solved +cliffs +##tree +resisted +anatomy +protesters +transparent +implied +##iga +injection +mattress +excluding +##mbo +defenses +helpless +devotion +##elli +growl +liberals +weber +phenomena +atoms +plug +##iff +mortality +apprentice +howe +convincing +aaa +swimmer +barber +leone +promptly +sodium +def +nowadays +arise +##oning +gloucester +corrected +dignity +norm +erie +##ders +elders +evacuated +sylvia +compression +##yar +hartford +pose +backpack +reasoning +accepts +24th +wipe +millimetres +marcel +##oda +dodgers +albion +1790 +overwhelmed +aerospace +oaks +1795 +showcase +acknowledge +recovering +nolan +ashe +hurts +geology +fashioned +disappearance +farewell +swollen +shrug +marquis +wimbledon +124 +rue +1792 +commemorate +reduces +experiencing +inevitable +calcutta +intel +##court +murderer +sticking +fisheries +imagery +bloom +280 +brake +##inus +gustav +hesitation +memorable +po +viral +beans +accidents +tunisia +antenna +spilled +consort +treatments +aye +perimeter +##gard +donation +hostage +migrated +banker +addiction +apex +lil +trout +##ously +conscience +##nova +rams +sands +genome +passionate +troubles +##lets +##set +amid +##ibility +##ret +higgins +exceed +vikings +##vie +payne +##zan +muscular +##ste +defendant +sucking +##wal +ibrahim +fuselage +claudia +vfl +europeans +snails +interval +##garh +preparatory +statewide +tasked +lacrosse +viktor +##lation +angola +##hra +flint +implications +employs +teens +patrons +stall +weekends +barriers +scrambled +nucleus +tehran +jenna +parsons +lifelong +robots +displacement +5000 +##bles +precipitation +##gt +knuckles +clutched +1802 +marrying +ecology +marx +accusations +declare +scars +kolkata +mat +meadows +bermuda +skeleton +finalists +vintage +crawl +coordinate +affects +subjected +orchestral +mistaken +##tc +mirrors +dipped +relied +260 +arches +candle +##nick +incorporating +wildly +fond +basilica +owl +fringe +rituals +whispering +stirred +feud +tertiary +slick +goat +honorable +whereby +skip +ricardo +stripes +parachute +adjoining +submerged +synthesizer +##gren +intend +positively +ninety +phi +beaver +partition +fellows +alexis +prohibition +carlisle +bizarre +fraternity +##bre +doubts +icy +cbc +aquatic +sneak +sonny +combines +airports +crude +supervised +spatial +merge +alfonso +##bic +corrupt +scan +undergo +##ams +disabilities +colombian +comparing +dolphins +perkins +##lish +reprinted +unanimous +bounced +hairs +underworld +midwest +semester +bucket +paperback +miniseries +coventry +demise +##leigh +demonstrations +sensor +rotating +yan +##hler +arrange +soils +##idge +hyderabad +labs +##dr +brakes +grandchildren +##nde +negotiated +rover +ferrari +continuation +directorate +augusta +stevenson +counterpart +gore +##rda +nursery +rican +ave +collectively +broadly +pastoral +repertoire +asserted +discovering +nordic +styled +fiba +cunningham +harley +middlesex +survives +tumor +tempo +zack +aiming +lok +urgent +##rade +##nto +devils +##ement +contractor +turin +##wl +##ool +bliss +repaired +simmons +moan +astronomical +cr +negotiate +lyric +1890s +lara +bred +clad +angus +pbs +##ience +engineered +posed +##lk +hernandez +possessions +elbows +psychiatric +strokes +confluence +electorate +lifts +campuses +lava +alps +##ep +##ution +##date +physicist +woody +##page +##ographic +##itis +juliet +reformation +sparhawk +320 +complement +suppressed +jewel +##½ +floated +##kas +continuity +sadly +##ische +inability +melting +scanning +paula +flour +judaism +safer +vague +##lm +solving +curb +##stown +financially +gable +bees +expired +miserable +cassidy +dominion +1789 +cupped +145 +robbery +facto +amos +warden +resume +tallest +marvin +ing +pounded +usd +declaring +gasoline +##aux +darkened +270 +650 +sophomore +##mere +erection +gossip +televised +risen +dial +##eu +pillars +##link +passages +profound +##tina +arabian +ashton +silicon +nail +##ead +##lated +##wer +##hardt +fleming +firearms +ducked +circuits +blows +waterloo +titans +##lina +atom +fireplace +cheshire +financed +activation +algorithms +##zzi +constituent +catcher +cherokee +partnerships +sexuality +platoon +tragic +vivian +guarded +whiskey +meditation +poetic +##late +##nga +##ake +porto +listeners +dominance +kendra +mona +chandler +factions +22nd +salisbury +attitudes +derivative +##ido +##haus +intake +paced +javier +illustrator +barrels +bias +cockpit +burnett +dreamed +ensuing +##anda +receptors +someday +hawkins +mattered +##lal +slavic +1799 +jesuit +cameroon +wasted +tai +wax +lowering +victorious +freaking +outright +hancock +librarian +sensing +bald +calcium +myers +tablet +announcing +barack +shipyard +pharmaceutical +##uan +greenwich +flush +medley +patches +wolfgang +pt +speeches +acquiring +exams +nikolai +##gg +hayden +kannada +##type +reilly +##pt +waitress +abdomen +devastated +capped +pseudonym +pharmacy +fulfill +paraguay +1796 +clicked +##trom +archipelago +syndicated +##hman +lumber +orgasm +rejection +clifford +lorraine +advent +mafia +rodney +brock +##ght +##used +##elia +cassette +chamberlain +despair +mongolia +sensors +developmental +upstream +##eg +##alis +spanning +165 +trombone +basque +seeded +interred +renewable +rhys +leapt +revision +molecule +##ages +chord +vicious +nord +shivered +23rd +arlington +debts +corpus +sunrise +bays +blackburn +centimetres +##uded +shuddered +gm +strangely +gripping +cartoons +isabelle +orbital +##ppa +seals +proving +##lton +refusal +strengthened +bust +assisting +baghdad +batsman +portrayal +mara +pushes +spears +og +##cock +reside +nathaniel +brennan +1776 +confirmation +caucus +##worthy +markings +yemen +nobles +ku +lazy +viewer +catalan +encompasses +sawyer +##fall +sparked +substances +patents +braves +arranger +evacuation +sergio +persuade +dover +tolerance +penguin +cum +jockey +insufficient +townships +occupying +declining +plural +processed +projection +puppet +flanders +introduces +liability +##yon +gymnastics +antwerp +taipei +hobart +candles +jeep +wes +observers +126 +chaplain +bundle +glorious +##hine +hazel +flung +sol +excavations +dumped +stares +sh +bangalore +triangular +icelandic +intervals +expressing +turbine +##vers +songwriting +crafts +##igo +jasmine +ditch +rite +##ways +entertaining +comply +sorrow +wrestlers +basel +emirates +marian +rivera +helpful +##some +caution +downward +networking +##atory +##tered +darted +genocide +emergence +replies +specializing +spokesman +convenient +unlocked +fading +augustine +concentrations +resemblance +elijah +investigator +andhra +##uda +promotes +bean +##rrell +fleeing +wan +simone +announcer +##ame +##bby +lydia +weaver +132 +residency +modification +##fest +stretches +##ast +alternatively +nat +lowe +lacks +##ented +pam +tile +concealed +inferior +abdullah +residences +tissues +vengeance +##ided +moisture +peculiar +groove +zip +bologna +jennings +ninja +oversaw +zombies +pumping +batch +livingston +emerald +installations +1797 +peel +nitrogen +rama +##fying +##star +schooling +strands +responding +werner +##ost +lime +casa +accurately +targeting +##rod +underway +##uru +hemisphere +lester +##yard +occupies +2d +griffith +angrily +reorganized +##owing +courtney +deposited +##dd +##30 +estadio +##ifies +dunn +exiled +##ying +checks +##combe +##о +##fly +successes +unexpectedly +blu +assessed +##flower +##ه +observing +sacked +spiders +kn +##tail +mu +nodes +prosperity +audrey +divisional +155 +broncos +tangled +adjust +feeds +erosion +paolo +surf +directory +snatched +humid +admiralty +screwed +gt +reddish +##nese +modules +trench +lamps +bind +leah +bucks +competes +##nz +##form +transcription +##uc +isles +violently +clutching +pga +cyclist +inflation +flats +ragged +unnecessary +##hian +stubborn +coordinated +harriet +baba +disqualified +330 +insect +wolfe +##fies +reinforcements +rocked +duel +winked +embraced +bricks +##raj +hiatus +defeats +pending +brightly +jealousy +##xton +##hm +##uki +lena +gdp +colorful +##dley +stein +kidney +##shu +underwear +wanderers +##haw +##icus +guardians +m³ +roared +habits +##wise +permits +gp +uranium +punished +disguise +bundesliga +elise +dundee +erotic +partisan +pi +collectors +float +individually +rendering +behavioral +bucharest +ser +hare +valerie +corporal +nutrition +proportional +##isa +immense +##kis +pavement +##zie +##eld +sutherland +crouched +1775 +##lp +suzuki +trades +endurance +operas +crosby +prayed +priory +rory +socially +##urn +gujarat +##pu +walton +cube +pasha +privilege +lennon +floods +thorne +waterfall +nipple +scouting +approve +##lov +minorities +voter +dwight +extensions +assure +ballroom +slap +dripping +privileges +rejoined +confessed +demonstrating +patriotic +yell +investor +##uth +pagan +slumped +squares +##cle +##kins +confront +bert +embarrassment +##aid +aston +urging +sweater +starr +yuri +brains +williamson +commuter +mortar +structured +selfish +exports +##jon +cds +##him +unfinished +##rre +mortgage +destinations +##nagar +canoe +solitary +buchanan +delays +magistrate +fk +##pling +motivation +##lier +##vier +recruiting +assess +##mouth +malik +antique +1791 +pius +rahman +reich +tub +zhou +smashed +airs +galway +xii +conditioning +honduras +discharged +dexter +##pf +lionel +129 +debates +lemon +tiffany +volunteered +dom +dioxide +procession +devi +sic +tremendous +advertisements +colts +transferring +verdict +hanover +decommissioned +utter +relate +pac +racism +##top +beacon +limp +similarity +terra +occurrence +ant +##how +becky +capt +updates +armament +richie +pal +##graph +halloween +mayo +##ssen +##bone +cara +serena +fcc +dolls +obligations +##dling +violated +lafayette +jakarta +exploitation +##ime +infamous +iconic +##lah +##park +kitty +moody +reginald +dread +spill +crystals +olivier +modeled +bluff +equilibrium +separating +notices +ordnance +extinction +onset +cosmic +attachment +sammy +expose +privy +anchored +##bil +abbott +admits +bending +baritone +emmanuel +policeman +vaughan +winged +climax +dresses +denny +polytechnic +mohamed +burmese +authentic +nikki +genetics +grandparents +homestead +gaza +postponed +metacritic +una +##sby +##bat +unstable +dissertation +##rial +##cian +curls +obscure +uncovered +bronx +praying +disappearing +##hoe +prehistoric +coke +turret +mutations +nonprofit +pits +monaco +##ي +##usion +prominently +dispatched +podium +##mir +uci +##uation +133 +fortifications +birthplace +kendall +##lby +##oll +preacher +rack +goodman +##rman +persistent +##ott +countless +jaime +recorder +lexington +persecution +jumps +renewal +wagons +##11 +crushing +##holder +decorations +##lake +abundance +wrath +laundry +£1 +garde +##rp +jeanne +beetles +peasant +##sl +splitting +caste +sergei +##rer +##ema +scripts +##ively +rub +satellites +##vor +inscribed +verlag +scrapped +gale +packages +chick +potato +slogan +kathleen +arabs +##culture +counterparts +reminiscent +choral +##tead +rand +retains +bushes +dane +accomplish +courtesy +closes +##oth +slaughter +hague +krakow +lawson +tailed +elias +ginger +##ttes +canopy +betrayal +rebuilding +turf +##hof +frowning +allegiance +brigades +kicks +rebuild +polls +alias +nationalism +td +rowan +audition +bowie +fortunately +recognizes +harp +dillon +horrified +##oro +renault +##tics +ropes +##α +presumed +rewarded +infrared +wiping +accelerated +illustration +##rid +presses +practitioners +badminton +##iard +detained +##tera +recognizing +relates +misery +##sies +##tly +reproduction +piercing +potatoes +thornton +esther +manners +hbo +##aan +ours +bullshit +ernie +perennial +sensitivity +illuminated +rupert +##jin +##iss +##ear +rfc +nassau +##dock +staggered +socialism +##haven +appointments +nonsense +prestige +sharma +haul +##tical +solidarity +gps +##ook +##rata +igor +pedestrian +##uit +baxter +tenants +wires +medication +unlimited +guiding +impacts +diabetes +##rama +sasha +pas +clive +extraction +131 +continually +constraints +##bilities +sonata +hunted +sixteenth +chu +planting +quote +mayer +pretended +abs +spat +##hua +ceramic +##cci +curtains +pigs +pitching +##dad +latvian +sore +dayton +##sted +##qi +patrols +slice +playground +##nted +shone +stool +apparatus +inadequate +mates +treason +##ija +desires +##liga +##croft +somalia +laurent +mir +leonardo +oracle +grape +obliged +chevrolet +thirteenth +stunning +enthusiastic +##ede +accounted +concludes +currents +basil +##kovic +drought +##rica +mai +##aire +shove +posting +##shed +pilgrimage +humorous +packing +fry +pencil +wines +smells +144 +marilyn +aching +newest +clung +bon +neighbours +sanctioned +##pie +mug +##stock +drowning +##mma +hydraulic +##vil +hiring +reminder +lilly +investigators +##ncies +sour +##eous +compulsory +packet +##rion +##graphic +##elle +cannes +##inate +depressed +##rit +heroic +importantly +theresa +##tled +conway +saturn +marginal +rae +##xia +corresponds +royce +pact +jasper +explosives +packaging +aluminium +##ttered +denotes +rhythmic +spans +assignments +hereditary +outlined +originating +sundays +lad +reissued +greeting +beatrice +##dic +pillar +marcos +plots +handbook +alcoholic +judiciary +avant +slides +extract +masculine +blur +##eum +##force +homage +trembled +owens +hymn +trey +omega +signaling +socks +accumulated +reacted +attic +theo +lining +angie +distraction +primera +talbot +##key +1200 +ti +creativity +billed +##hey +deacon +eduardo +identifies +proposition +dizzy +gunner +hogan +##yam +##pping +##hol +ja +##chan +jensen +reconstructed +##berger +clearance +darius +##nier +abe +harlem +plea +dei +circled +emotionally +notation +fascist +neville +exceeded +upwards +viable +ducks +##fo +workforce +racer +limiting +shri +##lson +possesses +1600 +kerr +moths +devastating +laden +disturbing +locking +##cture +gal +fearing +accreditation +flavor +aide +1870s +mountainous +##baum +melt +##ures +motel +texture +servers +soda +##mb +herd +##nium +erect +puzzled +hum +peggy +examinations +gould +testified +geoff +ren +devised +sacks +##law +denial +posters +grunted +cesar +tutor +ec +gerry +offerings +byrne +falcons +combinations +ct +incoming +pardon +rocking +26th +avengers +flared +mankind +seller +uttar +loch +nadia +stroking +exposing +##hd +fertile +ancestral +instituted +##has +noises +prophecy +taxation +eminent +vivid +pol +##bol +dart +indirect +multimedia +notebook +upside +displaying +adrenaline +referenced +geometric +##iving +progression +##ddy +blunt +announce +##far +implementing +##lav +aggression +liaison +cooler +cares +headache +plantations +gorge +dots +impulse +thickness +ashamed +averaging +kathy +obligation +precursor +137 +fowler +symmetry +thee +225 +hears +##rai +undergoing +ads +butcher +bowler +##lip +cigarettes +subscription +goodness +##ically +browne +##hos +##tech +kyoto +donor +##erty +damaging +friction +drifting +expeditions +hardened +prostitution +152 +fauna +blankets +claw +tossing +snarled +butterflies +recruits +investigative +coated +healed +138 +communal +hai +xiii +academics +boone +psychologist +restless +lahore +stephens +mba +brendan +foreigners +printer +##pc +ached +explode +27th +deed +scratched +dared +##pole +cardiac +1780 +okinawa +proto +commando +compelled +oddly +electrons +##base +replica +thanksgiving +##rist +sheila +deliberate +stafford +tidal +representations +hercules +ou +##path +##iated +kidnapping +lenses +##tling +deficit +samoa +mouths +consuming +computational +maze +granting +smirk +razor +fixture +ideals +inviting +aiden +nominal +##vs +issuing +julio +pitt +ramsey +docks +##oss +exhaust +##owed +bavarian +draped +anterior +mating +ethiopian +explores +noticing +##nton +discarded +convenience +hoffman +endowment +beasts +cartridge +mormon +paternal +probe +sleeves +interfere +lump +deadline +##rail +jenks +bulldogs +scrap +alternating +justified +reproductive +nam +seize +descending +secretariat +kirby +coupe +grouped +smash +panther +sedan +tapping +##18 +lola +cheer +germanic +unfortunate +##eter +unrelated +##fan +subordinate +##sdale +suzanne +advertisement +##ility +horsepower +##lda +cautiously +discourse +luigi +##mans +##fields +noun +prevalent +mao +schneider +everett +surround +governorate +kira +##avia +westward +##take +misty +rails +sustainability +134 +unused +##rating +packs +toast +unwilling +regulate +thy +suffrage +nile +awe +assam +definitions +travelers +affordable +##rb +conferred +sells +undefeated +beneficial +torso +basal +repeating +remixes +##pass +bahrain +cables +fang +##itated +excavated +numbering +statutory +##rey +deluxe +##lian +forested +ramirez +derbyshire +zeus +slamming +transfers +astronomer +banana +lottery +berg +histories +bamboo +##uchi +resurrection +posterior +bowls +vaguely +##thi +thou +preserving +tensed +offence +##inas +meyrick +callum +ridden +watt +langdon +tying +lowland +snorted +daring +truman +##hale +##girl +aura +overly +filing +weighing +goa +infections +philanthropist +saunders +eponymous +##owski +latitude +perspectives +reviewing +mets +commandant +radial +##kha +flashlight +reliability +koch +vowels +amazed +ada +elaine +supper +##rth +##encies +predator +debated +soviets +cola +##boards +##nah +compartment +crooked +arbitrary +fourteenth +##ctive +havana +majors +steelers +clips +profitable +ambush +exited +packers +##tile +nude +cracks +fungi +##е +limb +trousers +josie +shelby +tens +frederic +##ος +definite +smoothly +constellation +insult +baton +discs +lingering +##nco +conclusions +lent +staging +becker +grandpa +shaky +##tron +einstein +obstacles +sk +adverse +elle +economically +##moto +mccartney +thor +dismissal +motions +readings +nostrils +treatise +##pace +squeezing +evidently +prolonged +1783 +venezuelan +je +marguerite +beirut +takeover +shareholders +##vent +denise +digit +airplay +norse +##bbling +imaginary +pills +hubert +blaze +vacated +eliminating +##ello +vine +mansfield +##tty +retrospective +barrow +borne +clutch +bail +forensic +weaving +##nett +##witz +desktop +citadel +promotions +worrying +dorset +ieee +subdivided +##iating +manned +expeditionary +pickup +synod +chuckle +185 +barney +##rz +##ffin +functionality +karachi +litigation +meanings +uc +lick +turbo +anders +##ffed +execute +curl +oppose +ankles +typhoon +##د +##ache +##asia +linguistics +compassion +pressures +grazing +perfection +##iting +immunity +monopoly +muddy +backgrounds +136 +namibia +francesca +monitors +attracting +stunt +tuition +##ии +vegetable +##mates +##quent +mgm +jen +complexes +forts +##ond +cellar +bites +seventeenth +royals +flemish +failures +mast +charities +##cular +peruvian +capitals +macmillan +ipswich +outward +frigate +postgraduate +folds +employing +##ouse +concurrently +fiery +##tai +contingent +nightmares +monumental +nicaragua +##kowski +lizard +mal +fielding +gig +reject +##pad +harding +##ipe +coastline +##cin +##nos +beethoven +humphrey +innovations +##tam +##nge +norris +doris +solicitor +huang +obey +141 +##lc +niagara +##tton +shelves +aug +bourbon +curry +nightclub +specifications +hilton +##ndo +centennial +dispersed +worm +neglected +briggs +sm +font +kuala +uneasy +plc +##nstein +##bound +##aking +##burgh +awaiting +pronunciation +##bbed +##quest +eh +optimal +zhu +raped +greens +presided +brenda +worries +##life +venetian +marxist +turnout +##lius +refined +braced +sins +grasped +sunderland +nickel +speculated +lowell +cyrillic +communism +fundraising +resembling +colonists +mutant +freddie +usc +##mos +gratitude +##run +mural +##lous +chemist +wi +reminds +28th +steals +tess +pietro +##ingen +promoter +ri +microphone +honoured +rai +sant +##qui +feather +##nson +burlington +kurdish +terrorists +deborah +sickness +##wed +##eet +hazard +irritated +desperation +veil +clarity +##rik +jewels +xv +##gged +##ows +##cup +berkshire +unfair +mysteries +orchid +winced +exhaustion +renovations +stranded +obe +infinity +##nies +adapt +redevelopment +thanked +registry +olga +domingo +noir +tudor +ole +##atus +commenting +behaviors +##ais +crisp +pauline +probable +stirling +wigan +##bian +paralympics +panting +surpassed +##rew +luca +barred +pony +famed +##sters +cassandra +waiter +carolyn +exported +##orted +andres +destructive +deeds +jonah +castles +vacancy +suv +##glass +1788 +orchard +yep +famine +belarusian +sprang +##forth +skinny +##mis +administrators +rotterdam +zambia +zhao +boiler +discoveries +##ride +##physics +lucius +disappointing +outreach +spoon +##frame +qualifications +unanimously +enjoys +regency +##iidae +stade +realism +veterinary +rodgers +dump +alain +chestnut +castile +censorship +rumble +gibbs +##itor +communion +reggae +inactivated +logs +loads +##houses +homosexual +##iano +ale +informs +##cas +phrases +plaster +linebacker +ambrose +kaiser +fascinated +850 +limerick +recruitment +forge +mastered +##nding +leinster +rooted +threaten +##strom +borneo +##hes +suggestions +scholarships +propeller +documentaries +patronage +coats +constructing +invest +neurons +comet +entirety +shouts +identities +annoying +unchanged +wary +##antly +##ogy +neat +oversight +##kos +phillies +replay +constance +##kka +incarnation +humble +skies +minus +##acy +smithsonian +##chel +guerrilla +jar +cadets +##plate +surplus +audit +##aru +cracking +joanna +louisa +pacing +##lights +intentionally +##iri +diner +nwa +imprint +australians +tong +unprecedented +bunker +naive +specialists +ark +nichols +railing +leaked +pedal +##uka +shrub +longing +roofs +v8 +captains +neural +tuned +##ntal +##jet +emission +medina +frantic +codex +definitive +sid +abolition +intensified +stocks +enrique +sustain +genoa +oxide +##written +clues +cha +##gers +tributaries +fragment +venom +##rity +##ente +##sca +muffled +vain +sire +laos +##ingly +##hana +hastily +snapping +surfaced +sentiment +motive +##oft +contests +approximate +mesa +luckily +dinosaur +exchanges +propelled +accord +bourne +relieve +tow +masks +offended +##ues +cynthia +##mmer +rains +bartender +zinc +reviewers +lois +##sai +legged +arrogant +rafe +rosie +comprise +handicap +blockade +inlet +lagoon +copied +drilling +shelley +petals +##inian +mandarin +obsolete +##inated +onward +arguably +productivity +cindy +praising +seldom +busch +discusses +raleigh +shortage +ranged +stanton +encouragement +firstly +conceded +overs +temporal +##uke +cbe +##bos +woo +certainty +pumps +##pton +stalked +##uli +lizzie +periodic +thieves +weaker +##night +gases +shoving +chooses +wc +##chemical +prompting +weights +##kill +robust +flanked +sticky +hu +tuberculosis +##eb +##eal +christchurch +resembled +wallet +reese +inappropriate +pictured +distract +fixing +fiddle +giggled +burger +heirs +hairy +mechanic +torque +apache +obsessed +chiefly +cheng +logging +##tag +extracted +meaningful +numb +##vsky +gloucestershire +reminding +##bay +unite +##lit +breeds +diminished +clown +glove +1860s +##ن +##ug +archibald +focal +freelance +sliced +depiction +##yk +organism +switches +sights +stray +crawling +##ril +lever +leningrad +interpretations +loops +anytime +reel +alicia +delighted +##ech +inhaled +xiv +suitcase +bernie +vega +licenses +northampton +exclusion +induction +monasteries +racecourse +homosexuality +##right +##sfield +##rky +dimitri +michele +alternatives +ions +commentators +genuinely +objected +pork +hospitality +fencing +stephan +warships +peripheral +wit +drunken +wrinkled +quentin +spends +departing +chung +numerical +spokesperson +##zone +johannesburg +caliber +killers +##udge +assumes +neatly +demographic +abigail +bloc +##vel +mounting +##lain +bentley +slightest +xu +recipients +##jk +merlin +##writer +seniors +prisons +blinking +hindwings +flickered +kappa +##hel +80s +strengthening +appealing +brewing +gypsy +mali +lashes +hulk +unpleasant +harassment +bio +treaties +predict +instrumentation +pulp +troupe +boiling +mantle +##ffe +ins +##vn +dividing +handles +verbs +##onal +coconut +senegal +340 +thorough +gum +momentarily +##sto +cocaine +panicked +destined +##turing +teatro +denying +weary +captained +mans +##hawks +##code +wakefield +bollywood +thankfully +##16 +cyril +##wu +amendments +##bahn +consultation +stud +reflections +kindness +1787 +internally +##ovo +tex +mosaic +distribute +paddy +seeming +143 +##hic +piers +##15 +##mura +##verse +popularly +winger +kang +sentinel +mccoy +##anza +covenant +##bag +verge +fireworks +suppress +thrilled +dominate +##jar +swansea +##60 +142 +reconciliation +##ndi +stiffened +cue +dorian +##uf +damascus +amor +ida +foremost +##aga +porsche +unseen +dir +##had +##azi +stony +lexi +melodies +##nko +angular +integer +podcast +ants +inherent +jaws +justify +persona +##olved +josephine +##nr +##ressed +customary +flashes +gala +cyrus +glaring +backyard +ariel +physiology +greenland +html +stir +avon +atletico +finch +methodology +ked +##lent +mas +catholicism +townsend +branding +quincy +fits +containers +1777 +ashore +aragon +##19 +forearm +poisoning +##sd +adopting +conquer +grinding +amnesty +keller +finances +evaluate +forged +lankan +instincts +##uto +guam +bosnian +photographed +workplace +desirable +protector +##dog +allocation +intently +encourages +willy +##sten +bodyguard +electro +brighter +##ν +bihar +##chev +lasts +opener +amphibious +sal +verde +arte +##cope +captivity +vocabulary +yields +##tted +agreeing +desmond +pioneered +##chus +strap +campaigned +railroads +##ович +emblem +##dre +stormed +501 +##ulous +marijuana +northumberland +##gn +##nath +bowen +landmarks +beaumont +##qua +danube +##bler +attorneys +th +ge +flyers +critique +villains +cass +mutation +acc +##0s +colombo +mckay +motif +sampling +concluding +syndicate +##rell +neon +stables +ds +warnings +clint +mourning +wilkinson +##tated +merrill +leopard +evenings +exhaled +emil +sonia +ezra +discrete +stove +farrell +fifteenth +prescribed +superhero +##rier +worms +helm +wren +##duction +##hc +expo +##rator +hq +unfamiliar +antony +prevents +acceleration +fiercely +mari +painfully +calculations +cheaper +ign +clifton +irvine +davenport +mozambique +##np +pierced +##evich +wonders +##wig +##cate +##iling +crusade +ware +##uel +enzymes +reasonably +mls +##coe +mater +ambition +bunny +eliot +kernel +##fin +asphalt +headmaster +torah +aden +lush +pins +waived +##care +##yas +joao +substrate +enforce +##grad +##ules +alvarez +selections +epidemic +tempted +##bit +bremen +translates +ensured +waterfront +29th +forrest +manny +malone +kramer +reigning +cookies +simpler +absorption +205 +engraved +##ffy +evaluated +1778 +haze +146 +comforting +crossover +##abe +thorn +##rift +##imo +##pop +suppression +fatigue +cutter +##tr +201 +wurttemberg +##orf +enforced +hovering +proprietary +gb +samurai +syllable +ascent +lacey +tick +lars +tractor +merchandise +rep +bouncing +defendants +##yre +huntington +##ground +##oko +standardized +##hor +##hima +assassinated +nu +predecessors +rainy +liar +assurance +lyrical +##uga +secondly +flattened +ios +parameter +undercover +##mity +bordeaux +punish +ridges +markers +exodus +inactive +hesitate +debbie +nyc +pledge +savoy +nagar +offset +organist +##tium +hesse +marin +converting +##iver +diagram +propulsion +pu +validity +reverted +supportive +##dc +ministries +clans +responds +proclamation +##inae +##ø +##rea +ein +pleading +patriot +sf +birch +islanders +strauss +hates +##dh +brandenburg +concession +rd +##ob +1900s +killings +textbook +antiquity +cinematography +wharf +embarrassing +setup +creed +farmland +inequality +centred +signatures +fallon +370 +##ingham +##uts +ceylon +gazing +directive +laurie +##tern +globally +##uated +##dent +allah +excavation +threads +##cross +148 +frantically +icc +utilize +determines +respiratory +thoughtful +receptions +##dicate +merging +chandra +seine +147 +builders +builds +diagnostic +dev +visibility +goddamn +analyses +dhaka +cho +proves +chancel +concurrent +curiously +canadians +pumped +restoring +1850s +turtles +jaguar +sinister +spinal +traction +declan +vows +1784 +glowed +capitalism +swirling +install +universidad +##lder +##oat +soloist +##genic +##oor +coincidence +beginnings +nissan +dip +resorts +caucasus +combustion +infectious +##eno +pigeon +serpent +##itating +conclude +masked +salad +jew +##gr +surreal +toni +##wc +harmonica +151 +##gins +##etic +##coat +fishermen +intending +bravery +##wave +klaus +titan +wembley +taiwanese +ransom +40th +incorrect +hussein +eyelids +jp +cooke +dramas +utilities +##etta +##print +eisenhower +principally +granada +lana +##rak +openings +concord +##bl +bethany +connie +morality +sega +##mons +##nard +earnings +##kara +##cine +wii +communes +##rel +coma +composing +softened +severed +grapes +##17 +nguyen +analyzed +warlord +hubbard +heavenly +behave +slovenian +##hit +##ony +hailed +filmmakers +trance +caldwell +skye +unrest +coward +likelihood +##aging +bern +sci +taliban +honolulu +propose +##wang +1700 +browser +imagining +cobra +contributes +dukes +instinctively +conan +violinist +##ores +accessories +gradual +##amp +quotes +sioux +##dating +undertake +intercepted +sparkling +compressed +139 +fungus +tombs +haley +imposing +rests +degradation +lincolnshire +retailers +wetlands +tulsa +distributor +dungeon +nun +greenhouse +convey +atlantis +aft +exits +oman +dresser +lyons +##sti +joking +eddy +judgement +omitted +digits +##cts +##game +juniors +##rae +cents +stricken +une +##ngo +wizards +weir +breton +nan +technician +fibers +liking +royalty +##cca +154 +persia +terribly +magician +##rable +##unt +vance +cafeteria +booker +camille +warmer +##static +consume +cavern +gaps +compass +contemporaries +foyer +soothing +graveyard +maj +plunged +blush +##wear +cascade +demonstrates +ordinance +##nov +boyle +##lana +rockefeller +shaken +banjo +izzy +##ense +breathless +vines +##32 +##eman +alterations +chromosome +dwellings +feudal +mole +153 +catalonia +relics +tenant +mandated +##fm +fridge +hats +honesty +patented +raul +heap +cruisers +accusing +enlightenment +infants +wherein +chatham +contractors +zen +affinity +hc +osborne +piston +156 +traps +maturity +##rana +lagos +##zal +peering +##nay +attendant +dealers +protocols +subset +prospects +biographical +##cre +artery +##zers +insignia +nuns +endured +##eration +recommend +schwartz +serbs +berger +cromwell +crossroads +##ctor +enduring +clasped +grounded +##bine +marseille +twitched +abel +choke +https +catalyst +moldova +italians +##tist +disastrous +wee +##oured +##nti +wwf +nope +##piration +##asa +expresses +thumbs +167 +##nza +coca +1781 +cheating +##ption +skipped +sensory +heidelberg +spies +satan +dangers +semifinal +202 +bohemia +whitish +confusing +shipbuilding +relies +surgeons +landings +ravi +baku +moor +suffix +alejandro +##yana +litre +upheld +##unk +rajasthan +##rek +coaster +insists +posture +scenarios +etienne +favoured +appoint +transgender +elephants +poked +greenwood +defences +fulfilled +militant +somali +1758 +chalk +potent +##ucci +migrants +wink +assistants +nos +restriction +activism +niger +##ario +colon +shaun +##sat +daphne +##erated +swam +congregations +reprise +considerations +magnet +playable +xvi +##р +overthrow +tobias +knob +chavez +coding +##mers +propped +katrina +orient +newcomer +##suke +temperate +##pool +farmhouse +interrogation +##vd +committing +##vert +forthcoming +strawberry +joaquin +macau +ponds +shocking +siberia +##cellular +chant +contributors +##nant +##ologists +sped +absorb +hail +1782 +spared +##hore +barbados +karate +opus +originates +saul +##xie +evergreen +leaped +##rock +correlation +exaggerated +weekday +unification +bump +tracing +brig +afb +pathways +utilizing +##ners +mod +mb +disturbance +kneeling +##stad +##guchi +100th +pune +##thy +decreasing +168 +manipulation +miriam +academia +ecosystem +occupational +rbi +##lem +rift +##14 +rotary +stacked +incorporation +awakening +generators +guerrero +racist +##omy +cyber +derivatives +culminated +allie +annals +panzer +sainte +wikipedia +pops +zu +austro +##vate +algerian +politely +nicholson +mornings +educate +tastes +thrill +dartmouth +##gating +db +##jee +regan +differing +concentrating +choreography +divinity +##media +pledged +alexandre +routing +gregor +madeline +##idal +apocalypse +##hora +gunfire +culminating +elves +fined +liang +lam +programmed +tar +guessing +transparency +gabrielle +##gna +cancellation +flexibility +##lining +accession +shea +stronghold +nets +specializes +##rgan +abused +hasan +sgt +ling +exceeding +##₄ +admiration +supermarket +##ark +photographers +specialised +tilt +resonance +hmm +perfume +380 +sami +threatens +garland +botany +guarding +boiled +greet +puppy +russo +supplier +wilmington +vibrant +vijay +##bius +paralympic +grumbled +paige +faa +licking +margins +hurricanes +##gong +fest +grenade +ripping +##uz +counseling +weigh +##sian +needles +wiltshire +edison +costly +##not +fulton +tramway +redesigned +staffordshire +cache +gasping +watkins +sleepy +candidacy +##group +monkeys +timeline +throbbing +##bid +##sos +berth +uzbekistan +vanderbilt +bothering +overturned +ballots +gem +##iger +sunglasses +subscribers +hooker +compelling +ang +exceptionally +saloon +stab +##rdi +carla +terrifying +rom +##vision +coil +##oids +satisfying +vendors +31st +mackay +deities +overlooked +ambient +bahamas +felipe +olympia +whirled +botanist +advertised +tugging +##dden +disciples +morales +unionist +rites +foley +morse +motives +creepy +##₀ +soo +##sz +bargain +highness +frightening +turnpike +tory +reorganization +##cer +depict +biographer +##walk +unopposed +manifesto +##gles +institut +emile +accidental +kapoor +##dam +kilkenny +cortex +lively +##13 +romanesque +jain +shan +cannons +##ood +##ske +petrol +echoing +amalgamated +disappears +cautious +proposes +sanctions +trenton +##ر +flotilla +aus +contempt +tor +canary +cote +theirs +##hun +conceptual +deleted +fascinating +paso +blazing +elf +honourable +hutchinson +##eiro +##outh +##zin +surveyor +tee +amidst +wooded +reissue +intro +##ono +cobb +shelters +newsletter +hanson +brace +encoding +confiscated +dem +caravan +marino +scroll +melodic +cows +imam +##adi +##aneous +northward +searches +biodiversity +cora +310 +roaring +##bers +connell +theologian +halo +compose +pathetic +unmarried +dynamo +##oot +az +calculation +toulouse +deserves +humour +nr +forgiveness +tam +undergone +martyr +pamela +myths +whore +counselor +hicks +290 +heavens +battleship +electromagnetic +##bbs +stellar +establishments +presley +hopped +##chin +temptation +90s +wills +nas +##yuan +nhs +##nya +seminars +##yev +adaptations +gong +asher +lex +indicator +sikh +tobago +cites +goin +##yte +satirical +##gies +characterised +correspond +bubbles +lure +participates +##vid +eruption +skate +therapeutic +1785 +canals +wholesale +defaulted +sac +460 +petit +##zzled +virgil +leak +ravens +256 +portraying +##yx +ghetto +creators +dams +portray +vicente +##rington +fae +namesake +bounty +##arium +joachim +##ota +##iser +aforementioned +axle +snout +depended +dismantled +reuben +480 +##ibly +gallagher +##lau +##pd +earnest +##ieu +##iary +inflicted +objections +##llar +asa +gritted +##athy +jericho +##sea +##was +flick +underside +ceramics +undead +substituted +195 +eastward +undoubtedly +wheeled +chimney +##iche +guinness +cb +##ager +siding +##bell +traitor +baptiste +disguised +inauguration +149 +tipperary +choreographer +perched +warmed +stationary +eco +##ike +##ntes +bacterial +##aurus +flores +phosphate +##core +attacker +invaders +alvin +intersects +a1 +indirectly +immigrated +businessmen +cornelius +valves +narrated +pill +sober +ul +nationale +monastic +applicants +scenery +##jack +161 +motifs +constitutes +cpu +##osh +jurisdictions +sd +tuning +irritation +woven +##uddin +fertility +gao +##erie +antagonist +impatient +glacial +hides +boarded +denominations +interception +##jas +cookie +nicola +##tee +algebraic +marquess +bahn +parole +buyers +bait +turbines +paperwork +bestowed +natasha +renee +oceans +purchases +157 +vaccine +215 +##tock +fixtures +playhouse +integrate +jai +oswald +intellectuals +##cky +booked +nests +mortimer +##isi +obsession +sept +##gler +##sum +440 +scrutiny +simultaneous +squinted +##shin +collects +oven +shankar +penned +remarkably +##я +slips +luggage +spectral +1786 +collaborations +louie +consolidation +##ailed +##ivating +420 +hoover +blackpool +harness +ignition +vest +tails +belmont +mongol +skinner +##nae +visually +mage +derry +##tism +##unce +stevie +transitional +##rdy +redskins +drying +prep +prospective +##21 +annoyance +oversee +##loaded +fills +##books +##iki +announces +fda +scowled +respects +prasad +mystic +tucson +##vale +revue +springer +bankrupt +1772 +aristotle +salvatore +habsburg +##geny +dal +natal +nut +pod +chewing +darts +moroccan +walkover +rosario +lenin +punjabi +##ße +grossed +scattering +wired +invasive +hui +polynomial +corridors +wakes +gina +portrays +##cratic +arid +retreating +erich +irwin +sniper +##dha +linen +lindsey +maneuver +butch +shutting +socio +bounce +commemorative +postseason +jeremiah +pines +275 +mystical +beads +bp +abbas +furnace +bidding +consulted +assaulted +empirical +rubble +enclosure +sob +weakly +cancel +polly +yielded +##emann +curly +prediction +battered +70s +vhs +jacqueline +render +sails +barked +detailing +grayson +riga +sloane +raging +##yah +herbs +bravo +##athlon +alloy +giggle +imminent +suffers +assumptions +waltz +##itate +accomplishments +##ited +bathing +remixed +deception +prefix +##emia +deepest +##tier +##eis +balkan +frogs +##rong +slab +##pate +philosophers +peterborough +grains +imports +dickinson +rwanda +##atics +1774 +dirk +lan +tablets +##rove +clone +##rice +caretaker +hostilities +mclean +##gre +regimental +treasures +norms +impose +tsar +tango +diplomacy +variously +complain +192 +recognise +arrests +1779 +celestial +pulitzer +##dus +bing +libretto +##moor +adele +splash +##rite +expectation +lds +confronts +##izer +spontaneous +harmful +wedge +entrepreneurs +buyer +##ope +bilingual +translate +rugged +conner +circulated +uae +eaton +##gra +##zzle +lingered +lockheed +vishnu +reelection +alonso +##oom +joints +yankee +headline +cooperate +heinz +laureate +invading +##sford +echoes +scandinavian +##dham +hugging +vitamin +salute +micah +hind +trader +##sper +radioactive +##ndra +militants +poisoned +ratified +remark +campeonato +deprived +wander +prop +##dong +outlook +##tani +##rix +##eye +chiang +darcy +##oping +mandolin +spice +statesman +babylon +182 +walled +forgetting +afro +##cap +158 +giorgio +buffer +##polis +planetary +##gis +overlap +terminals +kinda +centenary +##bir +arising +manipulate +elm +ke +1770 +ak +##tad +chrysler +mapped +moose +pomeranian +quad +macarthur +assemblies +shoreline +recalls +stratford +##rted +noticeable +##evic +imp +##rita +##sque +accustomed +supplying +tents +disgusted +vogue +sipped +filters +khz +reno +selecting +luftwaffe +mcmahon +tyne +masterpiece +carriages +collided +dunes +exercised +flare +remembers +muzzle +##mobile +heck +##rson +burgess +lunged +middleton +boycott +bilateral +##sity +hazardous +lumpur +multiplayer +spotlight +jackets +goldman +liege +porcelain +rag +waterford +benz +attracts +hopeful +battling +ottomans +kensington +baked +hymns +cheyenne +lattice +levine +borrow +polymer +clashes +michaels +monitored +commitments +denounced +##25 +##von +cavity +##oney +hobby +akin +##holders +futures +intricate +cornish +patty +##oned +illegally +dolphin +##lag +barlow +yellowish +maddie +apologized +luton +plagued +##puram +nana +##rds +sway +fanny +łodz +##rino +psi +suspicions +hanged +##eding +initiate +charlton +##por +nak +competent +235 +analytical +annex +wardrobe +reservations +##rma +sect +162 +fairfax +hedge +piled +buckingham +uneven +bauer +simplicity +snyder +interpret +accountability +donors +moderately +byrd +continents +##cite +##max +disciple +hr +jamaican +ping +nominees +##uss +mongolian +diver +attackers +eagerly +ideological +pillows +miracles +apartheid +revolver +sulfur +clinics +moran +163 +##enko +ile +katy +rhetoric +##icated +chronology +recycling +##hrer +elongated +mughal +pascal +profiles +vibration +databases +domination +##fare +##rant +matthias +digest +rehearsal +polling +weiss +initiation +reeves +clinging +flourished +impress +ngo +##hoff +##ume +buckley +symposium +rhythms +weed +emphasize +transforming +##taking +##gence +##yman +accountant +analyze +flicker +foil +priesthood +voluntarily +decreases +##80 +##hya +slater +sv +charting +mcgill +##lde +moreno +##iu +besieged +zur +robes +##phic +admitting +api +deported +turmoil +peyton +earthquakes +##ares +nationalists +beau +clair +brethren +interrupt +welch +curated +galerie +requesting +164 +##ested +impending +steward +viper +##vina +complaining +beautifully +brandy +foam +nl +1660 +##cake +alessandro +punches +laced +explanations +##lim +attribute +clit +reggie +discomfort +##cards +smoothed +whales +##cene +adler +countered +duffy +disciplinary +widening +recipe +reliance +conducts +goats +gradient +preaching +##shaw +matilda +quasi +striped +meridian +cannabis +cordoba +certificates +##agh +##tering +graffiti +hangs +pilgrims +repeats +##ych +revive +urine +etat +##hawk +fueled +belts +fuzzy +susceptible +##hang +mauritius +salle +sincere +beers +hooks +##cki +arbitration +entrusted +advise +sniffed +seminar +junk +donnell +processors +principality +strapped +celia +mendoza +everton +fortunes +prejudice +starving +reassigned +steamer +##lund +tuck +evenly +foreman +##ffen +dans +375 +envisioned +slit +##xy +baseman +liberia +rosemary +##weed +electrified +periodically +potassium +stride +contexts +sperm +slade +mariners +influx +bianca +subcommittee +##rane +spilling +icao +estuary +##nock +delivers +iphone +##ulata +isa +mira +bohemian +dessert +##sbury +welcoming +proudly +slowing +##chs +musee +ascension +russ +##vian +waits +##psy +africans +exploit +##morphic +gov +eccentric +crab +peck +##ull +entrances +formidable +marketplace +groom +bolted +metabolism +patton +robbins +courier +payload +endure +##ifier +andes +refrigerator +##pr +ornate +##uca +ruthless +illegitimate +masonry +strasbourg +bikes +adobe +##³ +apples +quintet +willingly +niche +bakery +corpses +energetic +##cliffe +##sser +##ards +177 +centimeters +centro +fuscous +cretaceous +rancho +##yde +andrei +telecom +tottenham +oasis +ordination +vulnerability +presiding +corey +cp +penguins +sims +##pis +malawi +piss +##48 +correction +##cked +##ffle +##ryn +countdown +detectives +psychiatrist +psychedelic +dinosaurs +blouse +##get +choi +vowed +##oz +randomly +##pol +49ers +scrub +blanche +bruins +dusseldorf +##using +unwanted +##ums +212 +dominique +elevations +headlights +om +laguna +##oga +1750 +famously +ignorance +shrewsbury +##aine +ajax +breuning +che +confederacy +greco +overhaul +##screen +paz +skirts +disagreement +cruelty +jagged +phoebe +shifter +hovered +viruses +##wes +mandy +##lined +##gc +landlord +squirrel +dashed +##ι +ornamental +gag +wally +grange +literal +spurs +undisclosed +proceeding +yin +##text +billie +orphan +spanned +humidity +indy +weighted +presentations +explosions +lucian +##tary +vaughn +hindus +##anga +##hell +psycho +171 +daytona +protects +efficiently +rematch +sly +tandem +##oya +rebranded +impaired +hee +metropolis +peach +godfrey +diaspora +ethnicity +prosperous +gleaming +dar +grossing +playback +##rden +stripe +pistols +##tain +births +labelled +##cating +172 +rudy +alba +##onne +aquarium +hostility +##gb +##tase +shudder +sumatra +hardest +lakers +consonant +creeping +demos +homicide +capsule +zeke +liberties +expulsion +pueblo +##comb +trait +transporting +##ddin +##neck +##yna +depart +gregg +mold +ledge +hangar +oldham +playboy +termination +analysts +gmbh +romero +##itic +insist +cradle +filthy +brightness +slash +shootout +deposed +bordering +##truct +isis +microwave +tumbled +sheltered +cathy +werewolves +messy +andersen +convex +clapped +clinched +satire +wasting +edo +vc +rufus +##jak +mont +##etti +poznan +##keeping +restructuring +transverse +##rland +azerbaijani +slovene +gestures +roommate +choking +shear +##quist +vanguard +oblivious +##hiro +disagreed +baptism +##lich +coliseum +##aceae +salvage +societe +cory +locke +relocation +relying +versailles +ahl +swelling +##elo +cheerful +##word +##edes +gin +sarajevo +obstacle +diverted +##nac +messed +thoroughbred +fluttered +utrecht +chewed +acquaintance +assassins +dispatch +mirza +##wart +nike +salzburg +swell +yen +##gee +idle +ligue +samson +##nds +##igh +playful +spawned +##cise +tease +##case +burgundy +##bot +stirring +skeptical +interceptions +marathi +##dies +bedrooms +aroused +pinch +##lik +preferences +tattoos +buster +digitally +projecting +rust +##ital +kitten +priorities +addison +pseudo +##guard +dusk +icons +sermon +##psis +##iba +bt +##lift +##xt +ju +truce +rink +##dah +##wy +defects +psychiatry +offences +calculate +glucose +##iful +##rized +##unda +francaise +##hari +richest +warwickshire +carly +1763 +purity +redemption +lending +##cious +muse +bruises +cerebral +aero +carving +##name +preface +terminology +invade +monty +##int +anarchist +blurred +##iled +rossi +treats +guts +shu +foothills +ballads +undertaking +premise +cecilia +affiliates +blasted +conditional +wilder +minors +drone +rudolph +buffy +swallowing +horton +attested +##hop +rutherford +howell +primetime +livery +penal +##bis +minimize +hydro +wrecked +wrought +palazzo +##gling +cans +vernacular +friedman +nobleman +shale +walnut +danielle +##ection +##tley +sears +##kumar +chords +lend +flipping +streamed +por +dracula +gallons +sacrifices +gamble +orphanage +##iman +mckenzie +##gible +boxers +daly +##balls +##ان +208 +##ific +##rative +##iq +exploited +slated +##uity +circling +hillary +pinched +goldberg +provost +campaigning +lim +piles +ironically +jong +mohan +successors +usaf +##tem +##ught +autobiographical +haute +preserves +##ending +acquitted +comparisons +203 +hydroelectric +gangs +cypriot +torpedoes +rushes +chrome +derive +bumps +instability +fiat +pets +##mbe +silas +dye +reckless +settler +##itation +info +heats +##writing +176 +canonical +maltese +fins +mushroom +stacy +aspen +avid +##kur +##loading +vickers +gaston +hillside +statutes +wilde +gail +kung +sabine +comfortably +motorcycles +##rgo +169 +pneumonia +fetch +##sonic +axel +faintly +parallels +##oop +mclaren +spouse +compton +interdisciplinary +miner +##eni +181 +clamped +##chal +##llah +separates +versa +##mler +scarborough +labrador +##lity +##osing +rutgers +hurdles +como +166 +burt +divers +##100 +wichita +cade +coincided +##erson +bruised +mla +##pper +vineyard +##ili +##brush +notch +mentioning +jase +hearted +kits +doe +##acle +pomerania +##ady +ronan +seizure +pavel +problematic +##zaki +domenico +##ulin +catering +penelope +dependence +parental +emilio +ministerial +atkinson +##bolic +clarkson +chargers +colby +grill +peeked +arises +summon +##aged +fools +##grapher +faculties +qaeda +##vial +garner +refurbished +##hwa +geelong +disasters +nudged +bs +shareholder +lori +algae +reinstated +rot +##ades +##nous +invites +stainless +183 +inclusive +##itude +diocesan +til +##icz +denomination +##xa +benton +floral +registers +##ider +##erman +##kell +absurd +brunei +guangzhou +hitter +retaliation +##uled +##eve +blanc +nh +consistency +contamination +##eres +##rner +dire +palermo +broadcasters +diaries +inspire +vols +brewer +tightening +ky +mixtape +hormone +##tok +stokes +##color +##dly +##ssi +pg +##ometer +##lington +sanitation +##tility +intercontinental +apps +##adt +¹⁄₂ +cylinders +economies +favourable +unison +croix +gertrude +odyssey +vanity +dangling +##logists +upgrades +dice +middleweight +practitioner +##ight +206 +henrik +parlor +orion +angered +lac +python +blurted +##rri +sensual +intends +swings +angled +##phs +husky +attain +peerage +precinct +textiles +cheltenham +shuffled +dai +confess +tasting +bhutan +##riation +tyrone +segregation +abrupt +ruiz +##rish +smirked +blackwell +confidential +browning +amounted +##put +vase +scarce +fabulous +raided +staple +guyana +unemployed +glider +shay +##tow +carmine +troll +intervene +squash +superstar +##uce +cylindrical +len +roadway +researched +handy +##rium +##jana +meta +lao +declares +##rring +##tadt +##elin +##kova +willem +shrubs +napoleonic +realms +skater +qi +volkswagen +##ł +tad +hara +archaeologist +awkwardly +eerie +##kind +wiley +##heimer +##24 +titus +organizers +cfl +crusaders +lama +usb +vent +enraged +thankful +occupants +maximilian +##gaard +possessing +textbooks +##oran +collaborator +quaker +##ulo +avalanche +mono +silky +straits +isaiah +mustang +surged +resolutions +potomac +descend +cl +kilograms +plato +strains +saturdays +##olin +bernstein +##ype +holstein +ponytail +##watch +belize +conversely +heroine +perpetual +##ylus +charcoal +piedmont +glee +negotiating +backdrop +prologue +##jah +##mmy +pasadena +climbs +ramos +sunni +##holm +##tner +##tri +anand +deficiency +hertfordshire +stout +##avi +aperture +orioles +##irs +doncaster +intrigued +bombed +coating +otis +##mat +cocktail +##jit +##eto +amir +arousal +sar +##proof +##act +##ories +dixie +pots +##bow +whereabouts +159 +##fted +drains +bullying +cottages +scripture +coherent +fore +poe +appetite +##uration +sampled +##ators +##dp +derrick +rotor +jays +peacock +installment +##rro +advisors +##coming +rodeo +scotch +##mot +##db +##fen +##vant +ensued +rodrigo +dictatorship +martyrs +twenties +##н +towed +incidence +marta +rainforest +sai +scaled +##cles +oceanic +qualifiers +symphonic +mcbride +dislike +generalized +aubrey +colonization +##iation +##lion +##ssing +disliked +lublin +salesman +##ulates +spherical +whatsoever +sweating +avalon +contention +punt +severity +alderman +atari +##dina +##grant +##rop +scarf +seville +vertices +annexation +fairfield +fascination +inspiring +launches +palatinate +regretted +##rca +feral +##iom +elk +nap +olsen +reddy +yong +##leader +##iae +garment +transports +feng +gracie +outrage +viceroy +insides +##esis +breakup +grady +organizer +softer +grimaced +222 +murals +galicia +arranging +vectors +##rsten +bas +##sb +##cens +sloan +##eka +bitten +ara +fender +nausea +bumped +kris +banquet +comrades +detector +persisted +##llan +adjustment +endowed +cinemas +##shot +sellers +##uman +peek +epa +kindly +neglect +simpsons +talon +mausoleum +runaway +hangul +lookout +##cic +rewards +coughed +acquainted +chloride +##ald +quicker +accordion +neolithic +##qa +artemis +coefficient +lenny +pandora +tx +##xed +ecstasy +litter +segunda +chairperson +gemma +hiss +rumor +vow +nasal +antioch +compensate +patiently +transformers +##eded +judo +morrow +penis +posthumous +philips +bandits +husbands +denote +flaming +##any +##phones +langley +yorker +1760 +walters +##uo +##kle +gubernatorial +fatty +samsung +leroy +outlaw +##nine +unpublished +poole +jakob +##ᵢ +##ₙ +crete +distorted +superiority +##dhi +intercept +crust +mig +claus +crashes +positioning +188 +stallion +301 +frontal +armistice +##estinal +elton +aj +encompassing +camel +commemorated +malaria +woodward +calf +cigar +penetrate +##oso +willard +##rno +##uche +illustrate +amusing +convergence +noteworthy +##lma +##rva +journeys +realise +manfred +##sable +410 +##vocation +hearings +fiance +##posed +educators +provoked +adjusting +##cturing +modular +stockton +paterson +vlad +rejects +electors +selena +maureen +##tres +uber +##rce +swirled +##num +proportions +nanny +pawn +naturalist +parma +apostles +awoke +ethel +wen +##bey +monsoon +overview +##inating +mccain +rendition +risky +adorned +##ih +equestrian +germain +nj +conspicuous +confirming +##yoshi +shivering +##imeter +milestone +rumours +flinched +bounds +smacked +token +##bei +lectured +automobiles +##shore +impacted +##iable +nouns +nero +##leaf +ismail +prostitute +trams +##lace +bridget +sud +stimulus +impressions +reins +revolves +##oud +##gned +giro +honeymoon +##swell +criterion +##sms +##uil +libyan +prefers +##osition +211 +preview +sucks +accusation +bursts +metaphor +diffusion +tolerate +faye +betting +cinematographer +liturgical +specials +bitterly +humboldt +##ckle +flux +rattled +##itzer +archaeologists +odor +authorised +marshes +discretion +##ов +alarmed +archaic +inverse +##leton +explorers +##pine +drummond +tsunami +woodlands +##minate +##tland +booklet +insanity +owning +insert +crafted +calculus +##tore +receivers +##bt +stung +##eca +##nched +prevailing +travellers +eyeing +lila +graphs +##borne +178 +julien +##won +morale +adaptive +therapist +erica +cw +libertarian +bowman +pitches +vita +##ional +crook +##ads +##entation +caledonia +mutiny +##sible +1840s +automation +##ß +flock +##pia +ironic +pathology +##imus +remarried +##22 +joker +withstand +energies +##att +shropshire +hostages +madeleine +tentatively +conflicting +mateo +recipes +euros +ol +mercenaries +nico +##ndon +albuquerque +augmented +mythical +bel +freud +##child +cough +##lica +365 +freddy +lillian +genetically +nuremberg +calder +209 +bonn +outdoors +paste +suns +urgency +vin +restraint +tyson +##cera +##selle +barrage +bethlehem +kahn +##par +mounts +nippon +barony +happier +ryu +makeshift +sheldon +blushed +castillo +barking +listener +taped +bethel +fluent +headlines +pornography +rum +disclosure +sighing +mace +doubling +gunther +manly +##plex +rt +interventions +physiological +forwards +emerges +##tooth +##gny +compliment +rib +recession +visibly +barge +faults +connector +exquisite +prefect +##rlin +patio +##cured +elevators +brandt +italics +pena +173 +wasp +satin +ea +botswana +graceful +respectable +##jima +##rter +##oic +franciscan +generates +##dl +alfredo +disgusting +##olate +##iously +sherwood +warns +cod +promo +cheryl +sino +##ة +##escu +twitch +##zhi +brownish +thom +ortiz +##dron +densely +##beat +carmel +reinforce +##bana +187 +anastasia +downhill +vertex +contaminated +remembrance +harmonic +homework +##sol +fiancee +gears +olds +angelica +loft +ramsay +quiz +colliery +sevens +##cape +autism +##hil +walkway +##boats +ruben +abnormal +ounce +khmer +##bbe +zachary +bedside +morphology +punching +##olar +sparrow +convinces +##35 +hewitt +queer +remastered +rods +mabel +solemn +notified +lyricist +symmetric +##xide +174 +encore +passports +wildcats +##uni +baja +##pac +mildly +##ease +bleed +commodity +mounds +glossy +orchestras +##omo +damian +prelude +ambitions +##vet +awhile +remotely +##aud +asserts +imply +##iques +distinctly +modelling +remedy +##dded +windshield +dani +xiao +##endra +audible +powerplant +1300 +invalid +elemental +acquisitions +##hala +immaculate +libby +plata +smuggling +ventilation +denoted +minh +##morphism +430 +differed +dion +kelley +lore +mocking +sabbath +spikes +hygiene +drown +runoff +stylized +tally +liberated +aux +interpreter +righteous +aba +siren +reaper +pearce +millie +##cier +##yra +gaius +##iso +captures +##ttering +dorm +claudio +##sic +benches +knighted +blackness +##ored +discount +fumble +oxidation +routed +##ς +novak +perpendicular +spoiled +fracture +splits +##urt +pads +topology +##cats +axes +fortunate +offenders +protestants +esteem +221 +broadband +convened +frankly +hound +prototypes +isil +facilitated +keel +##sher +sahara +awaited +bubba +orb +prosecutors +186 +hem +520 +##xing +relaxing +remnant +romney +sorted +slalom +stefano +ulrich +##active +exemption +folder +pauses +foliage +hitchcock +epithet +204 +criticisms +##aca +ballistic +brody +hinduism +chaotic +youths +equals +##pala +pts +thicker +analogous +capitalist +improvised +overseeing +sinatra +ascended +beverage +##tl +straightforward +##kon +curran +##west +bois +325 +induce +surveying +emperors +sax +unpopular +##kk +cartoonist +fused +##mble +unto +##yuki +localities +##cko +##ln +darlington +slain +academie +lobbying +sediment +puzzles +##grass +defiance +dickens +manifest +tongues +alumnus +arbor +coincide +184 +appalachian +mustafa +examiner +cabaret +traumatic +yves +bracelet +draining +heroin +magnum +baths +odessa +consonants +mitsubishi +##gua +kellan +vaudeville +##fr +joked +null +straps +probation +##ław +ceded +interfaces +##pas +##zawa +blinding +viet +224 +rothschild +museo +640 +huddersfield +##vr +tactic +##storm +brackets +dazed +incorrectly +##vu +reg +glazed +fearful +manifold +benefited +irony +##sun +stumbling +##rte +willingness +balkans +mei +wraps +##aba +injected +##lea +gu +syed +harmless +##hammer +bray +takeoff +poppy +timor +cardboard +astronaut +purdue +weeping +southbound +cursing +stalls +diagonal +##neer +lamar +bryce +comte +weekdays +harrington +##uba +negatively +##see +lays +grouping +##cken +##henko +affirmed +halle +modernist +##lai +hodges +smelling +aristocratic +baptized +dismiss +justification +oilers +##now +coupling +qin +snack +healer +##qing +gardener +layla +battled +formulated +stephenson +gravitational +##gill +##jun +1768 +granny +coordinating +suites +##cd +##ioned +monarchs +##cote +##hips +sep +blended +apr +barrister +deposition +fia +mina +policemen +paranoid +##pressed +churchyard +covert +crumpled +creep +abandoning +tr +transmit +conceal +barr +understands +readiness +spire +##cology +##enia +##erry +610 +startling +unlock +vida +bowled +slots +##nat +##islav +spaced +trusting +admire +rig +##ink +slack +##70 +mv +207 +casualty +##wei +classmates +##odes +##rar +##rked +amherst +furnished +evolve +foundry +menace +mead +##lein +flu +wesleyan +##kled +monterey +webber +##vos +wil +##mith +##на +bartholomew +justices +restrained +##cke +amenities +191 +mediated +sewage +trenches +ml +mainz +##thus +1800s +##cula +##inski +caine +bonding +213 +converts +spheres +superseded +marianne +crypt +sweaty +ensign +historia +##br +spruce +##post +##ask +forks +thoughtfully +yukon +pamphlet +ames +##uter +karma +##yya +bryn +negotiation +sighs +incapable +##mbre +##ntial +actresses +taft +##mill +luce +prevailed +##amine +1773 +motionless +envoy +testify +investing +sculpted +instructors +provence +kali +cullen +horseback +##while +goodwin +##jos +gaa +norte +##ldon +modify +wavelength +abd +214 +skinned +sprinter +forecast +scheduling +marries +squared +tentative +##chman +boer +##isch +bolts +swap +fisherman +assyrian +impatiently +guthrie +martins +murdoch +194 +tanya +nicely +dolly +lacy +med +##45 +syn +decks +fashionable +millionaire +##ust +surfing +##ml +##ision +heaved +tammy +consulate +attendees +routinely +197 +fuse +saxophonist +backseat +malaya +##lord +scowl +tau +##ishly +193 +sighted +steaming +##rks +303 +911 +##holes +##hong +ching +##wife +bless +conserved +jurassic +stacey +unix +zion +chunk +rigorous +blaine +198 +peabody +slayer +dismay +brewers +nz +##jer +det +##glia +glover +postwar +int +penetration +sylvester +imitation +vertically +airlift +heiress +knoxville +viva +##uin +390 +macon +##rim +##fighter +##gonal +janice +##orescence +##wari +marius +belongings +leicestershire +196 +blanco +inverted +preseason +sanity +sobbing +##due +##elt +##dled +collingwood +regeneration +flickering +shortest +##mount +##osi +feminism +##lat +sherlock +cabinets +fumbled +northbound +precedent +snaps +##mme +researching +##akes +guillaume +insights +manipulated +vapor +neighbour +sap +gangster +frey +f1 +stalking +scarcely +callie +barnett +tendencies +audi +doomed +assessing +slung +panchayat +ambiguous +bartlett +##etto +distributing +violating +wolverhampton +##hetic +swami +histoire +##urus +liable +pounder +groin +hussain +larsen +popping +surprises +##atter +vie +curt +##station +mute +relocate +musicals +authorization +richter +##sef +immortality +tna +bombings +##press +deteriorated +yiddish +##acious +robbed +colchester +cs +pmid +ao +verified +balancing +apostle +swayed +recognizable +oxfordshire +retention +nottinghamshire +contender +judd +invitational +shrimp +uhf +##icient +cleaner +longitudinal +tanker +##mur +acronym +broker +koppen +sundance +suppliers +##gil +4000 +clipped +fuels +petite +##anne +landslide +helene +diversion +populous +landowners +auspices +melville +quantitative +##xes +ferries +nicky +##llus +doo +haunting +roche +carver +downed +unavailable +##pathy +approximation +hiroshima +##hue +garfield +valle +comparatively +keyboardist +traveler +##eit +congestion +calculating +subsidiaries +##bate +serb +modernization +fairies +deepened +ville +averages +##lore +inflammatory +tonga +##itch +co₂ +squads +##hea +gigantic +serum +enjoyment +retailer +verona +35th +cis +##phobic +magna +technicians +##vati +arithmetic +##sport +levin +##dation +amtrak +chow +sienna +##eyer +backstage +entrepreneurship +##otic +learnt +tao +##udy +worcestershire +formulation +baggage +hesitant +bali +sabotage +##kari +barren +enhancing +murmur +pl +freshly +putnam +syntax +aces +medicines +resentment +bandwidth +##sier +grins +chili +guido +##sei +framing +implying +gareth +lissa +genevieve +pertaining +admissions +geo +thorpe +proliferation +sato +bela +analyzing +parting +##gor +awakened +##isman +huddled +secrecy +##kling +hush +gentry +540 +dungeons +##ego +coasts +##utz +sacrificed +##chule +landowner +mutually +prevalence +programmer +adolescent +disrupted +seaside +gee +trusts +vamp +georgie +##nesian +##iol +schedules +sindh +##market +etched +hm +sparse +bey +beaux +scratching +gliding +unidentified +216 +collaborating +gems +jesuits +oro +accumulation +shaping +mbe +anal +##xin +231 +enthusiasts +newscast +##egan +janata +dewey +parkinson +179 +ankara +biennial +towering +dd +inconsistent +950 +##chet +thriving +terminate +cabins +furiously +eats +advocating +donkey +marley +muster +phyllis +leiden +##user +grassland +glittering +iucn +loneliness +217 +memorandum +armenians +##ddle +popularized +rhodesia +60s +lame +##illon +sans +bikini +header +orbits +##xx +##finger +##ulator +sharif +spines +biotechnology +strolled +naughty +yates +##wire +fremantle +milo +##mour +abducted +removes +##atin +humming +wonderland +##chrome +##ester +hume +pivotal +##rates +armand +grams +believers +elector +rte +apron +bis +scraped +##yria +endorsement +initials +##llation +eps +dotted +hints +buzzing +emigration +nearer +##tom +indicators +##ulu +coarse +neutron +protectorate +##uze +directional +exploits +pains +loire +1830s +proponents +guggenheim +rabbits +ritchie +305 +hectare +inputs +hutton +##raz +verify +##ako +boilers +longitude +##lev +skeletal +yer +emilia +citrus +compromised +##gau +pokemon +prescription +paragraph +eduard +cadillac +attire +categorized +kenyan +weddings +charley +##bourg +entertain +monmouth +##lles +nutrients +davey +mesh +incentive +practised +ecosystems +kemp +subdued +overheard +##rya +bodily +maxim +##nius +apprenticeship +ursula +##fight +lodged +rug +silesian +unconstitutional +patel +inspected +coyote +unbeaten +##hak +34th +disruption +convict +parcel +##cl +##nham +collier +implicated +mallory +##iac +##lab +susannah +winkler +##rber +shia +phelps +sediments +graphical +robotic +##sner +adulthood +mart +smoked +##isto +kathryn +clarified +##aran +divides +convictions +oppression +pausing +burying +##mt +federico +mathias +eileen +##tana +kite +hunched +##acies +189 +##atz +disadvantage +liza +kinetic +greedy +paradox +yokohama +dowager +trunks +ventured +##gement +gupta +vilnius +olaf +##thest +crimean +hopper +##ej +progressively +arturo +mouthed +arrondissement +##fusion +rubin +simulcast +oceania +##orum +##stra +##rred +busiest +intensely +navigator +cary +##vine +##hini +##bies +fife +rowe +rowland +posing +insurgents +shafts +lawsuits +activate +conor +inward +culturally +garlic +265 +##eering +eclectic +##hui +##kee +##nl +furrowed +vargas +meteorological +rendezvous +##aus +culinary +commencement +##dition +quota +##notes +mommy +salaries +overlapping +mule +##iology +##mology +sums +wentworth +##isk +##zione +mainline +subgroup +##illy +hack +plaintiff +verdi +bulb +differentiation +engagements +multinational +supplemented +bertrand +caller +regis +##naire +##sler +##arts +##imated +blossom +propagation +kilometer +viaduct +vineyards +##uate +beckett +optimization +golfer +songwriters +seminal +semitic +thud +volatile +evolving +ridley +##wley +trivial +distributions +scandinavia +jiang +##ject +wrestled +insistence +##dio +emphasizes +napkin +##ods +adjunct +rhyme +##ricted +##eti +hopeless +surrounds +tremble +32nd +smoky +##ntly +oils +medicinal +padded +steer +wilkes +219 +255 +concessions +hue +uniquely +blinded +landon +yahoo +##lane +hendrix +commemorating +dex +specify +chicks +##ggio +intercity +1400 +morley +##torm +highlighting +##oting +pang +oblique +stalled +##liner +flirting +newborn +1769 +bishopric +shaved +232 +currie +##ush +dharma +spartan +##ooped +favorites +smug +novella +sirens +abusive +creations +espana +##lage +paradigm +semiconductor +sheen +##rdo +##yen +##zak +nrl +renew +##pose +##tur +adjutant +marches +norma +##enity +ineffective +weimar +grunt +##gat +lordship +plotting +expenditure +infringement +lbs +refrain +av +mimi +mistakenly +postmaster +1771 +##bara +ras +motorsports +tito +199 +subjective +##zza +bully +stew +##kaya +prescott +1a +##raphic +##zam +bids +styling +paranormal +reeve +sneaking +exploding +katz +akbar +migrant +syllables +indefinitely +##ogical +destroys +replaces +applause +##phine +pest +##fide +218 +articulated +bertie +##thing +##cars +##ptic +courtroom +crowley +aesthetics +cummings +tehsil +hormones +titanic +dangerously +##ibe +stadion +jaenelle +auguste +ciudad +##chu +mysore +partisans +##sio +lucan +philipp +##aly +debating +henley +interiors +##rano +##tious +homecoming +beyonce +usher +henrietta +prepares +weeds +##oman +ely +plucked +##pire +##dable +luxurious +##aq +artifact +password +pasture +juno +maddy +minsk +##dder +##ologies +##rone +assessments +martian +royalist +1765 +examines +##mani +##rge +nino +223 +parry +scooped +relativity +##eli +##uting +##cao +congregational +noisy +traverse +##agawa +strikeouts +nickelodeon +obituary +transylvania +binds +depictions +polk +trolley +##yed +##lard +breeders +##under +dryly +hokkaido +1762 +strengths +stacks +bonaparte +connectivity +neared +prostitutes +stamped +anaheim +gutierrez +sinai +##zzling +bram +fresno +madhya +##86 +proton +##lena +##llum +##phon +reelected +wanda +##anus +##lb +ample +distinguishing +##yler +grasping +sermons +tomato +bland +stimulation +avenues +##eux +spreads +scarlett +fern +pentagon +assert +baird +chesapeake +ir +calmed +distortion +fatalities +##olis +correctional +pricing +##astic +##gina +prom +dammit +ying +collaborate +##chia +welterweight +33rd +pointer +substitution +bonded +umpire +communicating +multitude +paddle +##obe +federally +intimacy +##insky +betray +ssr +##lett +##lean +##lves +##therapy +airbus +##tery +functioned +ud +bearer +biomedical +netflix +##hire +##nca +condom +brink +ik +##nical +macy +##bet +flap +gma +experimented +jelly +lavender +##icles +##ulia +munro +##mian +##tial +rye +##rle +60th +gigs +hottest +rotated +predictions +fuji +bu +##erence +##omi +barangay +##fulness +##sas +clocks +##rwood +##liness +cereal +roe +wight +decker +uttered +babu +onion +xml +forcibly +##df +petra +sarcasm +hartley +peeled +storytelling +##42 +##xley +##ysis +##ffa +fibre +kiel +auditor +fig +harald +greenville +##berries +geographically +nell +quartz +##athic +cemeteries +##lr +crossings +nah +holloway +reptiles +chun +sichuan +snowy +660 +corrections +##ivo +zheng +ambassadors +blacksmith +fielded +fluids +hardcover +turnover +medications +melvin +academies +##erton +ro +roach +absorbing +spaniards +colton +##founded +outsider +espionage +kelsey +245 +edible +##ulf +dora +establishes +##sham +##tries +contracting +##tania +cinematic +costello +nesting +##uron +connolly +duff +##nology +mma +##mata +fergus +sexes +gi +optics +spectator +woodstock +banning +##hee +##fle +differentiate +outfielder +refinery +226 +312 +gerhard +horde +lair +drastically +##udi +landfall +##cheng +motorsport +odi +##achi +predominant +quay +skins +##ental +edna +harshly +complementary +murdering +##aves +wreckage +##90 +ono +outstretched +lennox +munitions +galen +reconcile +470 +scalp +bicycles +gillespie +questionable +rosenberg +guillermo +hostel +jarvis +kabul +volvo +opium +yd +##twined +abuses +decca +outpost +##cino +sensible +neutrality +##64 +ponce +anchorage +atkins +turrets +inadvertently +disagree +libre +vodka +reassuring +weighs +##yal +glide +jumper +ceilings +repertory +outs +stain +##bial +envy +##ucible +smashing +heightened +policing +hyun +mixes +lai +prima +##ples +celeste +##bina +lucrative +intervened +kc +manually +##rned +stature +staffed +bun +bastards +nairobi +priced +##auer +thatcher +##kia +tripped +comune +##ogan +##pled +brasil +incentives +emanuel +hereford +musica +##kim +benedictine +biennale +##lani +eureka +gardiner +rb +knocks +sha +##ael +##elled +##onate +efficacy +ventura +masonic +sanford +maize +leverage +##feit +capacities +santana +##aur +novelty +vanilla +##cter +##tour +benin +##oir +##rain +neptune +drafting +tallinn +##cable +humiliation +##boarding +schleswig +fabian +bernardo +liturgy +spectacle +sweeney +pont +routledge +##tment +cosmos +ut +hilt +sleek +universally +##eville +##gawa +typed +##dry +favors +allegheny +glaciers +##rly +recalling +aziz +##log +parasite +requiem +auf +##berto +##llin +illumination +##breaker +##issa +festivities +bows +govern +vibe +vp +333 +sprawled +larson +pilgrim +bwf +leaping +##rts +##ssel +alexei +greyhound +hoarse +##dler +##oration +seneca +##cule +gaping +##ulously +##pura +cinnamon +##gens +##rricular +craven +fantasies +houghton +engined +reigned +dictator +supervising +##oris +bogota +commentaries +unnatural +fingernails +spirituality +tighten +##tm +canadiens +protesting +intentional +cheers +sparta +##ytic +##iere +##zine +widen +belgarath +controllers +dodd +iaaf +navarre +##ication +defect +squire +steiner +whisky +##mins +560 +inevitably +tome +##gold +chew +##uid +##lid +elastic +##aby +streaked +alliances +jailed +regal +##ined +##phy +czechoslovak +narration +absently +##uld +bluegrass +guangdong +quran +criticizing +hose +hari +##liest +##owa +skier +streaks +deploy +##lom +raft +bose +dialed +huff +##eira +haifa +simplest +bursting +endings +ib +sultanate +##titled +franks +whitman +ensures +sven +##ggs +collaborators +forster +organising +ui +banished +napier +injustice +teller +layered +thump +##otti +roc +battleships +evidenced +fugitive +sadie +robotics +##roud +equatorial +geologist +##iza +yielding +##bron +##sr +internationale +mecca +##diment +sbs +skyline +toad +uploaded +reflective +undrafted +lal +leafs +bayern +##dai +lakshmi +shortlisted +##stick +##wicz +camouflage +donate +af +christi +lau +##acio +disclosed +nemesis +1761 +assemble +straining +northamptonshire +tal +##asi +bernardino +premature +heidi +42nd +coefficients +galactic +reproduce +buzzed +sensations +zionist +monsieur +myrtle +##eme +archery +strangled +musically +viewpoint +antiquities +bei +trailers +seahawks +cured +pee +preferring +tasmanian +lange +sul +##mail +##working +colder +overland +lucivar +massey +gatherings +haitian +##smith +disapproval +flaws +##cco +##enbach +1766 +npr +##icular +boroughs +creole +forums +techno +1755 +dent +abdominal +streetcar +##eson +##stream +procurement +gemini +predictable +##tya +acheron +christoph +feeder +fronts +vendor +bernhard +jammu +tumors +slang +##uber +goaltender +twists +curving +manson +vuelta +mer +peanut +confessions +pouch +unpredictable +allowance +theodor +vascular +##factory +bala +authenticity +metabolic +coughing +nanjing +##cea +pembroke +##bard +splendid +36th +ff +hourly +##ahu +elmer +handel +##ivate +awarding +thrusting +dl +experimentation +##hesion +##46 +caressed +entertained +steak +##rangle +biologist +orphans +baroness +oyster +stepfather +##dridge +mirage +reefs +speeding +##31 +barons +1764 +227 +inhabit +preached +repealed +##tral +honoring +boogie +captives +administer +johanna +##imate +gel +suspiciously +1767 +sobs +##dington +backbone +hayward +garry +##folding +##nesia +maxi +##oof +##ppe +ellison +galileo +##stand +crimea +frenzy +amour +bumper +matrices +natalia +baking +garth +palestinians +##grove +smack +conveyed +ensembles +gardening +##manship +##rup +##stituting +1640 +harvesting +topography +jing +shifters +dormitory +##carriage +##lston +ist +skulls +##stadt +dolores +jewellery +sarawak +##wai +##zier +fences +christy +confinement +tumbling +credibility +fir +stench +##bria +##plication +##nged +##sam +virtues +##belt +marjorie +pba +##eem +##made +celebrates +schooner +agitated +barley +fulfilling +anthropologist +##pro +restrict +novi +regulating +##nent +padres +##rani +##hesive +loyola +tabitha +milky +olson +proprietor +crambidae +guarantees +intercollegiate +ljubljana +hilda +##sko +ignorant +hooded +##lts +sardinia +##lidae +##vation +frontman +privileged +witchcraft +##gp +jammed +laude +poking +##than +bracket +amazement +yunnan +##erus +maharaja +linnaeus +264 +commissioning +milano +peacefully +##logies +akira +rani +regulator +##36 +grasses +##rance +luzon +crows +compiler +gretchen +seaman +edouard +tab +buccaneers +ellington +hamlets +whig +socialists +##anto +directorial +easton +mythological +##kr +##vary +rhineland +semantic +taut +dune +inventions +succeeds +##iter +replication +branched +##pired +jul +prosecuted +kangaroo +penetrated +##avian +middlesbrough +doses +bleak +madam +predatory +relentless +##vili +reluctance +##vir +hailey +crore +silvery +1759 +monstrous +swimmers +transmissions +hawthorn +informing +##eral +toilets +caracas +crouch +kb +##sett +295 +cartel +hadley +##aling +alexia +yvonne +##biology +cinderella +eton +superb +blizzard +stabbing +industrialist +maximus +##gm +##orus +groves +maud +clade +oversized +comedic +##bella +rosen +nomadic +fulham +montane +beverages +galaxies +redundant +swarm +##rot +##folia +##llis +buckinghamshire +fen +bearings +bahadur +##rom +gilles +phased +dynamite +faber +benoit +vip +##ount +##wd +booking +fractured +tailored +anya +spices +westwood +cairns +auditions +inflammation +steamed +##rocity +##acion +##urne +skyla +thereof +watford +torment +archdeacon +transforms +lulu +demeanor +fucked +serge +##sor +mckenna +minas +entertainer +##icide +caress +originate +residue +##sty +1740 +##ilised +##org +beech +##wana +subsidies +##ghton +emptied +gladstone +ru +firefighters +voodoo +##rcle +het +nightingale +tamara +edmond +ingredient +weaknesses +silhouette +285 +compatibility +withdrawing +hampson +##mona +anguish +giggling +##mber +bookstore +##jiang +southernmost +tilting +##vance +bai +economical +rf +briefcase +dreadful +hinted +projections +shattering +totaling +##rogate +analogue +indicted +periodical +fullback +##dman +haynes +##tenberg +##ffs +##ishment +1745 +thirst +stumble +penang +vigorous +##ddling +##kor +##lium +octave +##ove +##enstein +##inen +##ones +siberian +##uti +cbn +repeal +swaying +##vington +khalid +tanaka +unicorn +otago +plastered +lobe +riddle +##rella +perch +##ishing +croydon +filtered +graeme +tripoli +##ossa +crocodile +##chers +sufi +mined +##tung +inferno +lsu +##phi +swelled +utilizes +£2 +cale +periodicals +styx +hike +informally +coop +lund +##tidae +ala +hen +qui +transformations +disposed +sheath +chickens +##cade +fitzroy +sas +silesia +unacceptable +odisha +1650 +sabrina +pe +spokane +ratios +athena +massage +shen +dilemma +##drum +##riz +##hul +corona +doubtful +niall +##pha +##bino +fines +cite +acknowledging +bangor +ballard +bathurst +##resh +huron +mustered +alzheimer +garments +kinase +tyre +warship +##cp +flashback +pulmonary +braun +cheat +kamal +cyclists +constructions +grenades +ndp +traveller +excuses +stomped +signalling +trimmed +futsal +mosques +relevance +##wine +wta +##23 +##vah +##lter +hoc +##riding +optimistic +##´s +deco +sim +interacting +rejecting +moniker +waterways +##ieri +##oku +mayors +gdansk +outnumbered +pearls +##ended +##hampton +fairs +totals +dominating +262 +notions +stairway +compiling +pursed +commodities +grease +yeast +##jong +carthage +griffiths +residual +amc +contraction +laird +sapphire +##marine +##ivated +amalgamation +dissolve +inclination +lyle +packaged +altitudes +suez +canons +graded +lurched +narrowing +boasts +guise +wed +enrico +##ovsky +rower +scarred +bree +cub +iberian +protagonists +bargaining +proposing +trainers +voyages +vans +fishes +##aea +##ivist +##verance +encryption +artworks +kazan +sabre +cleopatra +hepburn +rotting +supremacy +mecklenburg +##brate +burrows +hazards +outgoing +flair +organizes +##ctions +scorpion +##usions +boo +234 +chevalier +dunedin +slapping +##34 +ineligible +pensions +##38 +##omic +manufactures +emails +bismarck +238 +weakening +blackish +ding +mcgee +quo +##rling +northernmost +xx +manpower +greed +sampson +clicking +##ange +##horpe +##inations +##roving +torre +##eptive +##moral +symbolism +38th +asshole +meritorious +outfits +splashed +biographies +sprung +astros +##tale +302 +737 +filly +raoul +nw +tokugawa +linden +clubhouse +##apa +tracts +romano +##pio +putin +tags +##note +chained +dickson +gunshot +moe +gunn +rashid +##tails +zipper +##bas +##nea +contrasted +##ply +##udes +plum +pharaoh +##pile +aw +comedies +ingrid +sandwiches +subdivisions +1100 +mariana +nokia +kamen +hz +delaney +veto +herring +##words +possessive +outlines +##roup +siemens +stairwell +rc +gallantry +messiah +palais +yells +233 +zeppelin +##dm +bolivar +##cede +smackdown +mckinley +##mora +##yt +muted +geologic +finely +unitary +avatar +hamas +maynard +rees +bog +contrasting +##rut +liv +chico +disposition +pixel +##erate +becca +dmitry +yeshiva +narratives +##lva +##ulton +mercenary +sharpe +tempered +navigate +stealth +amassed +keynes +##lini +untouched +##rrie +havoc +lithium +##fighting +abyss +graf +southward +wolverine +balloons +implements +ngos +transitions +##icum +ambushed +concacaf +dormant +economists +##dim +costing +csi +rana +universite +boulders +verity +##llon +collin +mellon +misses +cypress +fluorescent +lifeless +spence +##ulla +crewe +shepard +pak +revelations +##م +jolly +gibbons +paw +##dro +##quel +freeing +##test +shack +fries +palatine +##51 +##hiko +accompaniment +cruising +recycled +##aver +erwin +sorting +synthesizers +dyke +realities +sg +strides +enslaved +wetland +##ghan +competence +gunpowder +grassy +maroon +reactors +objection +##oms +carlson +gearbox +macintosh +radios +shelton +##sho +clergyman +prakash +254 +mongols +trophies +oricon +228 +stimuli +twenty20 +cantonese +cortes +mirrored +##saurus +bhp +cristina +melancholy +##lating +enjoyable +nuevo +##wny +downfall +schumacher +##ind +banging +lausanne +rumbled +paramilitary +reflex +ax +amplitude +migratory +##gall +##ups +midi +barnard +lastly +sherry +##hp +##nall +keystone +##kra +carleton +slippery +##53 +coloring +foe +socket +otter +##rgos +mats +##tose +consultants +bafta +bison +topping +##km +490 +primal +abandonment +transplant +atoll +hideous +mort +pained +reproduced +tae +howling +##turn +unlawful +billionaire +hotter +poised +lansing +##chang +dinamo +retro +messing +nfc +domesday +##mina +blitz +timed +##athing +##kley +ascending +gesturing +##izations +signaled +tis +chinatown +mermaid +savanna +jameson +##aint +catalina +##pet +##hers +cochrane +cy +chatting +##kus +alerted +computation +mused +noelle +majestic +mohawk +campo +octagonal +##sant +##hend +241 +aspiring +##mart +comprehend +iona +paralyzed +shimmering +swindon +rhone +##eley +reputed +configurations +pitchfork +agitation +francais +gillian +lipstick +##ilo +outsiders +pontifical +resisting +bitterness +sewer +rockies +##edd +##ucher +misleading +1756 +exiting +galloway +##nging +risked +##heart +246 +commemoration +schultz +##rka +integrating +##rsa +poses +shrieked +##weiler +guineas +gladys +jerking +owls +goldsmith +nightly +penetrating +##unced +lia +##33 +ignited +betsy +##aring +##thorpe +follower +vigorously +##rave +coded +kiran +knit +zoology +tbilisi +##28 +##bered +repository +govt +deciduous +dino +growling +##bba +enhancement +unleashed +chanting +pussy +biochemistry +##eric +kettle +repression +toxicity +nrhp +##arth +##kko +##bush +ernesto +commended +outspoken +242 +mca +parchment +sms +kristen +##aton +bisexual +raked +glamour +navajo +a2 +conditioned +showcased +##hma +spacious +youthful +##esa +usl +appliances +junta +brest +layne +conglomerate +enchanted +chao +loosened +picasso +circulating +inspect +montevideo +##centric +##kti +piazza +spurred +##aith +bari +freedoms +poultry +stamford +lieu +##ect +indigo +sarcastic +bahia +stump +attach +dvds +frankenstein +lille +approx +scriptures +pollen +##script +nmi +overseen +##ivism +tides +proponent +newmarket +inherit +milling +##erland +centralized +##rou +distributors +credentials +drawers +abbreviation +##lco +##xon +downing +uncomfortably +ripe +##oes +erase +franchises +##ever +populace +##bery +##khar +decomposition +pleas +##tet +daryl +sabah +##stle +##wide +fearless +genie +lesions +annette +##ogist +oboe +appendix +nair +dripped +petitioned +maclean +mosquito +parrot +rpg +hampered +1648 +operatic +reservoirs +##tham +irrelevant +jolt +summarized +##fp +medallion +##taff +##− +clawed +harlow +narrower +goddard +marcia +bodied +fremont +suarez +altering +tempest +mussolini +porn +##isms +sweetly +oversees +walkers +solitude +grimly +shrines +hk +ich +supervisors +hostess +dietrich +legitimacy +brushes +expressive +##yp +dissipated +##rse +localized +systemic +##nikov +gettysburg +##js +##uaries +dialogues +muttering +251 +housekeeper +sicilian +discouraged +##frey +beamed +kaladin +halftime +kidnap +##amo +##llet +1754 +synonymous +depleted +instituto +insulin +reprised +##opsis +clashed +##ctric +interrupting +radcliffe +insisting +medici +1715 +ejected +playfully +turbulent +##47 +starvation +##rini +shipment +rebellious +petersen +verification +merits +##rified +cakes +##charged +1757 +milford +shortages +spying +fidelity +##aker +emitted +storylines +harvested +seismic +##iform +cheung +kilda +theoretically +barbie +lynx +##rgy +##tius +goblin +mata +poisonous +##nburg +reactive +residues +obedience +##евич +conjecture +##rac +401 +hating +sixties +kicker +moaning +motown +##bha +emancipation +neoclassical +##hering +consoles +ebert +professorship +##tures +sustaining +assaults +obeyed +affluent +incurred +tornadoes +##eber +##zow +emphasizing +highlanders +cheated +helmets +##ctus +internship +terence +bony +executions +legislators +berries +peninsular +tinged +##aco +1689 +amplifier +corvette +ribbons +lavish +pennant +##lander +worthless +##chfield +##forms +mariano +pyrenees +expenditures +##icides +chesterfield +mandir +tailor +39th +sergey +nestled +willed +aristocracy +devotees +goodnight +raaf +rumored +weaponry +remy +appropriations +harcourt +burr +riaa +##lence +limitation +unnoticed +guo +soaking +swamps +##tica +collapsing +tatiana +descriptive +brigham +psalm +##chment +maddox +##lization +patti +caliph +##aja +akron +injuring +serra +##ganj +basins +##sari +astonished +launcher +##church +hilary +wilkins +sewing +##sf +stinging +##fia +##ncia +underwood +startup +##ition +compilations +vibrations +embankment +jurist +##nity +bard +juventus +groundwater +kern +palaces +helium +boca +cramped +marissa +soto +##worm +jae +princely +##ggy +faso +bazaar +warmly +##voking +229 +pairing +##lite +##grate +##nets +wien +freaked +ulysses +rebirth +##alia +##rent +mummy +guzman +jimenez +stilled +##nitz +trajectory +tha +woken +archival +professions +##pts +##pta +hilly +shadowy +shrink +##bolt +norwood +glued +migrate +stereotypes +devoid +##pheus +625 +evacuate +horrors +infancy +gotham +knowles +optic +downloaded +sachs +kingsley +parramatta +darryl +mor +##onale +shady +commence +confesses +kan +##meter +##placed +marlborough +roundabout +regents +frigates +io +##imating +gothenburg +revoked +carvings +clockwise +convertible +intruder +##sche +banged +##ogo +vicky +bourgeois +##mony +dupont +footing +##gum +pd +##real +buckle +yun +penthouse +sane +720 +serviced +stakeholders +neumann +bb +##eers +comb +##gam +catchment +pinning +rallies +typing +##elles +forefront +freiburg +sweetie +giacomo +widowed +goodwill +worshipped +aspirations +midday +##vat +fishery +##trick +bournemouth +turk +243 +hearth +ethanol +guadalajara +murmurs +sl +##uge +afforded +scripted +##hta +wah +##jn +coroner +translucent +252 +memorials +puck +progresses +clumsy +##race +315 +candace +recounted +##27 +##slin +##uve +filtering +##mac +howl +strata +heron +leveled +##ays +dubious +##oja +##т +##wheel +citations +exhibiting +##laya +##mics +##pods +turkic +##lberg +injunction +##ennial +##mit +antibodies +##44 +organise +##rigues +cardiovascular +cushion +inverness +##zquez +dia +cocoa +sibling +##tman +##roid +expanse +feasible +tunisian +algiers +##relli +rus +bloomberg +dso +westphalia +bro +tacoma +281 +downloads +##ours +konrad +duran +##hdi +continuum +jett +compares +legislator +secession +##nable +##gues +##zuka +translating +reacher +##gley +##ła +aleppo +##agi +tc +orchards +trapping +linguist +versatile +drumming +postage +calhoun +superiors +##mx +barefoot +leary +##cis +ignacio +alfa +kaplan +##rogen +bratislava +mori +##vot +disturb +haas +313 +cartridges +gilmore +radiated +salford +tunic +hades +##ulsive +archeological +delilah +magistrates +auditioned +brewster +charters +empowerment +blogs +cappella +dynasties +iroquois +whipping +##krishna +raceway +truths +myra +weaken +judah +mcgregor +##horse +mic +refueling +37th +burnley +bosses +markus +premio +query +##gga +dunbar +##economic +darkest +lyndon +sealing +commendation +reappeared +##mun +addicted +ezio +slaughtered +satisfactory +shuffle +##eves +##thic +##uj +fortification +warrington +##otto +resurrected +fargo +mane +##utable +##lei +##space +foreword +ox +##aris +##vern +abrams +hua +##mento +sakura +##alo +uv +sentimental +##skaya +midfield +##eses +sturdy +scrolls +macleod +##kyu +entropy +##lance +mitochondrial +cicero +excelled +thinner +convoys +perceive +##oslav +##urable +systematically +grind +burkina +287 +##tagram +ops +##aman +guantanamo +##cloth +##tite +forcefully +wavy +##jou +pointless +##linger +##tze +layton +portico +superficial +clerical +outlaws +##hism +burials +muir +##inn +creditors +hauling +rattle +##leg +calais +monde +archers +reclaimed +dwell +wexford +hellenic +falsely +remorse +##tek +dough +furnishings +##uttered +gabon +neurological +novice +##igraphy +contemplated +pulpit +nightstand +saratoga +##istan +documenting +pulsing +taluk +##firmed +busted +marital +##rien +disagreements +wasps +##yes +hodge +mcdonnell +mimic +fran +pendant +dhabi +musa +##nington +congratulations +argent +darrell +concussion +losers +regrets +thessaloniki +reversal +donaldson +hardwood +thence +achilles +ritter +##eran +demonic +jurgen +prophets +goethe +eki +classmate +buff +##cking +yank +irrational +##inging +perished +seductive +qur +sourced +##crat +##typic +mustard +ravine +barre +horizontally +characterization +phylogenetic +boise +##dit +##runner +##tower +brutally +intercourse +seduce +##bbing +fay +ferris +ogden +amar +nik +unarmed +##inator +evaluating +kyrgyzstan +sweetness +##lford +##oki +mccormick +meiji +notoriety +stimulate +disrupt +figuring +instructional +mcgrath +##zoo +groundbreaking +##lto +flinch +khorasan +agrarian +bengals +mixer +radiating +##sov +ingram +pitchers +nad +tariff +##cript +tata +##codes +##emi +##ungen +appellate +lehigh +##bled +##giri +brawl +duct +texans +##ciation +##ropolis +skipper +speculative +vomit +doctrines +stresses +253 +davy +graders +whitehead +jozef +timely +cumulative +haryana +paints +appropriately +boon +cactus +##ales +##pid +dow +legions +##pit +perceptions +1730 +picturesque +##yse +periphery +rune +wr +##aha +celtics +sentencing +whoa +##erin +confirms +variance +425 +moines +mathews +spade +rave +m1 +fronted +fx +blending +alleging +reared +##gl +237 +##paper +grassroots +eroded +##free +##physical +directs +ordeal +##sław +accelerate +hacker +rooftop +##inia +lev +buys +cebu +devote +##lce +specialising +##ulsion +choreographed +repetition +warehouses +##ryl +paisley +tuscany +analogy +sorcerer +hash +huts +shards +descends +exclude +nix +chaplin +gaga +ito +vane +##drich +causeway +misconduct +limo +orchestrated +glands +jana +##kot +u2 +##mple +##sons +branching +contrasts +scoop +longed +##virus +chattanooga +##75 +syrup +cornerstone +##tized +##mind +##iaceae +careless +precedence +frescoes +##uet +chilled +consult +modelled +snatch +peat +##thermal +caucasian +humane +relaxation +spins +temperance +##lbert +occupations +lambda +hybrids +moons +mp3 +##oese +247 +rolf +societal +yerevan +ness +##ssler +befriended +mechanized +nominate +trough +boasted +cues +seater +##hom +bends +##tangle +conductors +emptiness +##lmer +eurasian +adriatic +tian +##cie +anxiously +lark +propellers +chichester +jock +ev +2a +##holding +credible +recounts +tori +loyalist +abduction +##hoot +##redo +nepali +##mite +ventral +tempting +##ango +##crats +steered +##wice +javelin +dipping +laborers +prentice +looming +titanium +##ː +badges +emir +tensor +##ntation +egyptians +rash +denies +hawthorne +lombard +showers +wehrmacht +dietary +trojan +##reus +welles +executing +horseshoe +lifeboat +##lak +elsa +infirmary +nearing +roberta +boyer +mutter +trillion +joanne +##fine +##oked +sinks +vortex +uruguayan +clasp +sirius +##block +accelerator +prohibit +sunken +byu +chronological +diplomats +ochreous +510 +symmetrical +1644 +maia +##tology +salts +reigns +atrocities +##ия +hess +bared +issn +##vyn +cater +saturated +##cycle +##isse +sable +voyager +dyer +yusuf +##inge +fountains +wolff +##39 +##nni +engraving +rollins +atheist +ominous +##ault +herr +chariot +martina +strung +##fell +##farlane +horrific +sahib +gazes +saetan +erased +ptolemy +##olic +flushing +lauderdale +analytic +##ices +530 +navarro +beak +gorilla +herrera +broom +guadalupe +raiding +sykes +311 +bsc +deliveries +1720 +invasions +carmichael +tajikistan +thematic +ecumenical +sentiments +onstage +##rians +##brand +##sume +catastrophic +flanks +molten +##arns +waller +aimee +terminating +##icing +alternately +##oche +nehru +printers +outraged +##eving +empires +template +banners +repetitive +za +##oise +vegetarian +##tell +guiana +opt +cavendish +lucknow +synthesized +##hani +##mada +finalized +##ctable +fictitious +mayoral +unreliable +##enham +embracing +peppers +rbis +##chio +##neo +inhibition +slashed +togo +orderly +embroidered +safari +salty +236 +barron +benito +totaled +##dak +pubs +simulated +caden +devin +tolkien +momma +welding +sesame +##ept +gottingen +hardness +630 +shaman +temeraire +620 +adequately +pediatric +##kit +ck +assertion +radicals +composure +cadence +seafood +beaufort +lazarus +mani +warily +cunning +kurdistan +249 +cantata +##kir +ares +##41 +##clusive +nape +townland +geared +insulted +flutter +boating +violate +draper +dumping +malmo +##hh +##romatic +firearm +alta +bono +obscured +##clave +exceeds +panorama +unbelievable +##train +preschool +##essed +disconnected +installing +rescuing +secretaries +accessibility +##castle +##drive +##ifice +##film +bouts +slug +waterway +mindanao +##buro +##ratic +halves +##ل +calming +liter +maternity +adorable +bragg +electrification +mcc +##dote +roxy +schizophrenia +##body +munoz +kaye +whaling +239 +mil +tingling +tolerant +##ago +unconventional +volcanoes +##finder +deportivo +##llie +robson +kaufman +neuroscience +wai +deportation +masovian +scraping +converse +##bh +hacking +bulge +##oun +administratively +yao +580 +amp +mammoth +booster +claremont +hooper +nomenclature +pursuits +mclaughlin +melinda +##sul +catfish +barclay +substrates +taxa +zee +originals +kimberly +packets +padma +##ality +borrowing +ostensibly +solvent +##bri +##genesis +##mist +lukas +shreveport +veracruz +##ь +##lou +##wives +cheney +tt +anatolia +hobbs +##zyn +cyclic +radiant +alistair +greenish +siena +dat +independents +##bation +conform +pieter +hyper +applicant +bradshaw +spores +telangana +vinci +inexpensive +nuclei +322 +jang +nme +soho +spd +##ign +cradled +receptionist +pow +##43 +##rika +fascism +##ifer +experimenting +##ading +##iec +##region +345 +jocelyn +maris +stair +nocturnal +toro +constabulary +elgin +##kker +msc +##giving +##schen +##rase +doherty +doping +sarcastically +batter +maneuvers +##cano +##apple +##gai +##git +intrinsic +##nst +##stor +1753 +showtime +cafes +gasps +lviv +ushered +##thed +fours +restart +astonishment +transmitting +flyer +shrugs +##sau +intriguing +cones +dictated +mushrooms +medial +##kovsky +##elman +escorting +gaped +##26 +godfather +##door +##sell +djs +recaptured +timetable +vila +1710 +3a +aerodrome +mortals +scientology +##orne +angelina +mag +convection +unpaid +insertion +intermittent +lego +##nated +endeavor +kota +pereira +##lz +304 +bwv +glamorgan +insults +agatha +fey +##cend +fleetwood +mahogany +protruding +steamship +zeta +##arty +mcguire +suspense +##sphere +advising +urges +##wala +hurriedly +meteor +gilded +inline +arroyo +stalker +##oge +excitedly +revered +##cure +earle +introductory +##break +##ilde +mutants +puff +pulses +reinforcement +##haling +curses +lizards +stalk +correlated +##fixed +fallout +macquarie +##unas +bearded +denton +heaving +802 +##ocation +winery +assign +dortmund +##lkirk +everest +invariant +charismatic +susie +##elling +bled +lesley +telegram +sumner +bk +##ogen +##к +wilcox +needy +colbert +duval +##iferous +##mbled +allotted +attends +imperative +##hita +replacements +hawker +##inda +insurgency +##zee +##eke +casts +##yla +680 +ives +transitioned +##pack +##powering +authoritative +baylor +flex +cringed +plaintiffs +woodrow +##skie +drastic +ape +aroma +unfolded +commotion +nt +preoccupied +theta +routines +lasers +privatization +wand +domino +ek +clenching +nsa +strategically +showered +bile +handkerchief +pere +storing +christophe +insulting +316 +nakamura +romani +asiatic +magdalena +palma +cruises +stripping +405 +konstantin +soaring +##berman +colloquially +forerunner +havilland +incarcerated +parasites +sincerity +##utus +disks +plank +saigon +##ining +corbin +homo +ornaments +powerhouse +##tlement +chong +fastened +feasibility +idf +morphological +usable +##nish +##zuki +aqueduct +jaguars +keepers +##flies +aleksandr +faust +assigns +ewing +bacterium +hurled +tricky +hungarians +integers +wallis +321 +yamaha +##isha +hushed +oblivion +aviator +evangelist +friars +##eller +monograph +ode +##nary +airplanes +labourers +charms +##nee +1661 +hagen +tnt +rudder +fiesta +transcript +dorothea +ska +inhibitor +maccabi +retorted +raining +encompassed +clauses +menacing +1642 +lineman +##gist +vamps +##ape +##dick +gloom +##rera +dealings +easing +seekers +##nut +##pment +helens +unmanned +##anu +##isson +basics +##amy +##ckman +adjustments +1688 +brutality +horne +##zell +sui +##55 +##mable +aggregator +##thal +rhino +##drick +##vira +counters +zoom +##01 +##rting +mn +montenegrin +packard +##unciation +##♭ +##kki +reclaim +scholastic +thugs +pulsed +##icia +syriac +quan +saddam +banda +kobe +blaming +buddies +dissent +##lusion +##usia +corbett +jaya +delle +erratic +lexie +##hesis +435 +amiga +hermes +##pressing +##leen +chapels +gospels +jamal +##uating +compute +revolving +warp +##sso +##thes +armory +##eras +##gol +antrim +loki +##kow +##asian +##good +##zano +braid +handwriting +subdistrict +funky +pantheon +##iculate +concurrency +estimation +improper +juliana +##his +newcomers +johnstone +staten +communicated +##oco +##alle +sausage +stormy +##stered +##tters +superfamily +##grade +acidic +collateral +tabloid +##oped +##rza +bladder +austen +##ellant +mcgraw +##hay +hannibal +mein +aquino +lucifer +wo +badger +boar +cher +christensen +greenberg +interruption +##kken +jem +244 +mocked +bottoms +cambridgeshire +##lide +sprawling +##bbly +eastwood +ghent +synth +##buck +advisers +##bah +nominally +hapoel +qu +daggers +estranged +fabricated +towels +vinnie +wcw +misunderstanding +anglia +nothin +unmistakable +##dust +##lova +chilly +marquette +truss +##edge +##erine +reece +##lty +##chemist +##connected +272 +308 +41st +bash +raion +waterfalls +##ump +##main +labyrinth +queue +theorist +##istle +bharatiya +flexed +soundtracks +rooney +leftist +patrolling +wharton +plainly +alleviate +eastman +schuster +topographic +engages +immensely +unbearable +fairchild +1620 +dona +lurking +parisian +oliveira +ia +indictment +hahn +bangladeshi +##aster +vivo +##uming +##ential +antonia +expects +indoors +kildare +harlan +##logue +##ogenic +##sities +forgiven +##wat +childish +tavi +##mide +##orra +plausible +grimm +successively +scooted +##bola +##dget +##rith +spartans +emery +flatly +azure +epilogue +##wark +flourish +##iny +##tracted +##overs +##oshi +bestseller +distressed +receipt +spitting +hermit +topological +##cot +drilled +subunit +francs +##layer +eel +##fk +##itas +octopus +footprint +petitions +ufo +##say +##foil +interfering +leaking +palo +##metry +thistle +valiant +##pic +narayan +mcpherson +##fast +gonzales +##ym +##enne +dustin +novgorod +solos +##zman +doin +##raph +##patient +##meyer +soluble +ashland +cuffs +carole +pendleton +whistling +vassal +##river +deviation +revisited +constituents +rallied +rotate +loomed +##eil +##nting +amateurs +augsburg +auschwitz +crowns +skeletons +##cona +bonnet +257 +dummy +globalization +simeon +sleeper +mandal +differentiated +##crow +##mare +milne +bundled +exasperated +talmud +owes +segregated +##feng +##uary +dentist +piracy +props +##rang +devlin +##torium +malicious +paws +##laid +dependency +##ergy +##fers +##enna +258 +pistons +rourke +jed +grammatical +tres +maha +wig +512 +ghostly +jayne +##achal +##creen +##ilis +##lins +##rence +designate +##with +arrogance +cambodian +clones +showdown +throttle +twain +##ception +lobes +metz +nagoya +335 +braking +##furt +385 +roaming +##minster +amin +crippled +##37 +##llary +indifferent +hoffmann +idols +intimidating +1751 +261 +influenza +memo +onions +1748 +bandage +consciously +##landa +##rage +clandestine +observes +swiped +tangle +##ener +##jected +##trum +##bill +##lta +hugs +congresses +josiah +spirited +##dek +humanist +managerial +filmmaking +inmate +rhymes +debuting +grimsby +ur +##laze +duplicate +vigor +##tf +republished +bolshevik +refurbishment +antibiotics +martini +methane +newscasts +royale +horizons +levant +iain +visas +##ischen +paler +##around +manifestation +snuck +alf +chop +futile +pedestal +rehab +##kat +bmg +kerman +res +fairbanks +jarrett +abstraction +saharan +##zek +1746 +procedural +clearer +kincaid +sash +luciano +##ffey +crunch +helmut +##vara +revolutionaries +##tute +creamy +leach +##mmon +1747 +permitting +nes +plight +wendell +##lese +contra +ts +clancy +ipa +mach +staples +autopsy +disturbances +nueva +karin +pontiac +##uding +proxy +venerable +haunt +leto +bergman +expands +##helm +wal +##pipe +canning +celine +cords +obesity +##enary +intrusion +planner +##phate +reasoned +sequencing +307 +harrow +##chon +##dora +marred +mcintyre +repay +tarzan +darting +248 +harrisburg +margarita +repulsed +##hur +##lding +belinda +hamburger +novo +compliant +runways +bingham +registrar +skyscraper +ic +cuthbert +improvisation +livelihood +##corp +##elial +admiring +##dened +sporadic +believer +casablanca +popcorn +##29 +asha +shovel +##bek +##dice +coiled +tangible +##dez +casper +elsie +resin +tenderness +rectory +##ivision +avail +sonar +##mori +boutique +##dier +guerre +bathed +upbringing +vaulted +sandals +blessings +##naut +##utnant +1680 +306 +foxes +pia +corrosion +hesitantly +confederates +crystalline +footprints +shapiro +tirana +valentin +drones +45th +microscope +shipments +texted +inquisition +wry +guernsey +unauthorized +resigning +760 +ripple +schubert +stu +reassure +felony +##ardo +brittle +koreans +##havan +##ives +dun +implicit +tyres +##aldi +##lth +magnolia +##ehan +##puri +##poulos +aggressively +fei +gr +familiarity +##poo +indicative +##trust +fundamentally +jimmie +overrun +395 +anchors +moans +##opus +britannia +armagh +##ggle +purposely +seizing +##vao +bewildered +mundane +avoidance +cosmopolitan +geometridae +quartermaster +caf +415 +chatter +engulfed +gleam +purge +##icate +juliette +jurisprudence +guerra +revisions +##bn +casimir +brew +##jm +1749 +clapton +cloudy +conde +hermitage +278 +simulations +torches +vincenzo +matteo +##rill +hidalgo +booming +westbound +accomplishment +tentacles +unaffected +##sius +annabelle +flopped +sloping +##litz +dreamer +interceptor +vu +##loh +consecration +copying +messaging +breaker +climates +hospitalized +1752 +torino +afternoons +winfield +witnessing +##teacher +breakers +choirs +sawmill +coldly +##ege +sipping +haste +uninhabited +conical +bibliography +pamphlets +severn +edict +##oca +deux +illnesses +grips +##pl +rehearsals +sis +thinkers +tame +##keepers +1690 +acacia +reformer +##osed +##rys +shuffling +##iring +##shima +eastbound +ionic +rhea +flees +littered +##oum +rocker +vomiting +groaning +champ +overwhelmingly +civilizations +paces +sloop +adoptive +##tish +skaters +##vres +aiding +mango +##joy +nikola +shriek +##ignon +pharmaceuticals +##mg +tuna +calvert +gustavo +stocked +yearbook +##urai +##mana +computed +subsp +riff +hanoi +kelvin +hamid +moors +pastures +summons +jihad +nectar +##ctors +bayou +untitled +pleasing +vastly +republics +intellect +##η +##ulio +##tou +crumbling +stylistic +sb +##ی +consolation +frequented +h₂o +walden +widows +##iens +404 +##ignment +chunks +improves +288 +grit +recited +##dev +snarl +sociological +##arte +##gul +inquired +##held +bruise +clube +consultancy +homogeneous +hornets +multiplication +pasta +prick +savior +##grin +##kou +##phile +yoon +##gara +grimes +vanishing +cheering +reacting +bn +distillery +##quisite +##vity +coe +dockyard +massif +##jord +escorts +voss +##valent +byte +chopped +hawke +illusions +workings +floats +##koto +##vac +kv +annapolis +madden +##onus +alvaro +noctuidae +##cum +##scopic +avenge +steamboat +forte +illustrates +erika +##trip +570 +dew +nationalities +bran +manifested +thirsty +diversified +muscled +reborn +##standing +arson +##lessness +##dran +##logram +##boys +##kushima +##vious +willoughby +##phobia +286 +alsace +dashboard +yuki +##chai +granville +myspace +publicized +tricked +##gang +adjective +##ater +relic +reorganisation +enthusiastically +indications +saxe +##lassified +consolidate +iec +padua +helplessly +ramps +renaming +regulars +pedestrians +accents +convicts +inaccurate +lowers +mana +##pati +barrie +bjp +outta +someplace +berwick +flanking +invoked +marrow +sparsely +excerpts +clothed +rei +##ginal +wept +##straße +##vish +alexa +excel +##ptive +membranes +aquitaine +creeks +cutler +sheppard +implementations +ns +##dur +fragrance +budge +concordia +magnesium +marcelo +##antes +gladly +vibrating +##rral +##ggles +montrose +##omba +lew +seamus +1630 +cocky +##ament +##uen +bjorn +##rrick +fielder +fluttering +##lase +methyl +kimberley +mcdowell +reductions +barbed +##jic +##tonic +aeronautical +condensed +distracting +##promising +huffed +##cala +##sle +claudius +invincible +missy +pious +balthazar +ci +##lang +butte +combo +orson +##dication +myriad +1707 +silenced +##fed +##rh +coco +netball +yourselves +##oza +clarify +heller +peg +durban +etudes +offender +roast +blackmail +curvature +##woods +vile +309 +illicit +suriname +##linson +overture +1685 +bubbling +gymnast +tucking +##mming +##ouin +maldives +##bala +gurney +##dda +##eased +##oides +backside +pinto +jars +racehorse +tending +##rdial +baronetcy +wiener +duly +##rke +barbarian +cupping +flawed +##thesis +bertha +pleistocene +puddle +swearing +##nob +##tically +fleeting +prostate +amulet +educating +##mined +##iti +##tler +75th +jens +respondents +analytics +cavaliers +papacy +raju +##iente +##ulum +##tip +funnel +271 +disneyland +##lley +sociologist +##iam +2500 +faulkner +louvre +menon +##dson +276 +##ower +afterlife +mannheim +peptide +referees +comedians +meaningless +##anger +##laise +fabrics +hurley +renal +sleeps +##bour +##icle +breakout +kristin +roadside +animator +clover +disdain +unsafe +redesign +##urity +firth +barnsley +portage +reset +narrows +268 +commandos +expansive +speechless +tubular +##lux +essendon +eyelashes +smashwords +##yad +##bang +##claim +craved +sprinted +chet +somme +astor +wrocław +orton +266 +bane +##erving +##uing +mischief +##amps +##sund +scaling +terre +##xious +impairment +offenses +undermine +moi +soy +contiguous +arcadia +inuit +seam +##tops +macbeth +rebelled +##icative +##iot +590 +elaborated +frs +uniformed +##dberg +259 +powerless +priscilla +stimulated +980 +qc +arboretum +frustrating +trieste +bullock +##nified +enriched +glistening +intern +##adia +locus +nouvelle +ollie +ike +lash +starboard +ee +tapestry +headlined +hove +rigged +##vite +pollock +##yme +thrive +clustered +cas +roi +gleamed +olympiad +##lino +pressured +regimes +##hosis +##lick +ripley +##ophone +kickoff +gallon +rockwell +##arable +crusader +glue +revolutions +scrambling +1714 +grover +##jure +englishman +aztec +263 +contemplating +coven +ipad +preach +triumphant +tufts +##esian +rotational +##phus +328 +falkland +##brates +strewn +clarissa +rejoin +environmentally +glint +banded +drenched +moat +albanians +johor +rr +maestro +malley +nouveau +shaded +taxonomy +v6 +adhere +bunk +airfields +##ritan +1741 +encompass +remington +tran +##erative +amelie +mazda +friar +morals +passions +##zai +breadth +vis +##hae +argus +burnham +caressing +insider +rudd +##imov +##mini +##rso +italianate +murderous +textual +wainwright +armada +bam +weave +timer +##taken +##nh +fra +##crest +ardent +salazar +taps +tunis +##ntino +allegro +gland +philanthropic +##chester +implication +##optera +esq +judas +noticeably +wynn +##dara +inched +indexed +crises +villiers +bandit +royalties +patterned +cupboard +interspersed +accessory +isla +kendrick +entourage +stitches +##esthesia +headwaters +##ior +interlude +distraught +draught +1727 +##basket +biased +sy +transient +triad +subgenus +adapting +kidd +shortstop +##umatic +dimly +spiked +mcleod +reprint +nellie +pretoria +windmill +##cek +singled +##mps +273 +reunite +##orous +747 +bankers +outlying +##omp +##ports +##tream +apologies +cosmetics +patsy +##deh +##ocks +##yson +bender +nantes +serene +##nad +lucha +mmm +323 +##cius +##gli +cmll +coinage +nestor +juarez +##rook +smeared +sprayed +twitching +sterile +irina +embodied +juveniles +enveloped +miscellaneous +cancers +dq +gulped +luisa +crested +swat +donegal +ref +##anov +##acker +hearst +mercantile +##lika +doorbell +ua +vicki +##alla +##som +bilbao +psychologists +stryker +sw +horsemen +turkmenistan +wits +##national +anson +mathew +screenings +##umb +rihanna +##agne +##nessy +aisles +##iani +##osphere +hines +kenton +saskatoon +tasha +truncated +##champ +##itan +mildred +advises +fredrik +interpreting +inhibitors +##athi +spectroscopy +##hab +##kong +karim +panda +##oia +##nail +##vc +conqueror +kgb +leukemia +##dity +arrivals +cheered +pisa +phosphorus +shielded +##riated +mammal +unitarian +urgently +chopin +sanitary +##mission +spicy +drugged +hinges +##tort +tipping +trier +impoverished +westchester +##caster +267 +epoch +nonstop +##gman +##khov +aromatic +centrally +cerro +##tively +##vio +billions +modulation +sedimentary +283 +facilitating +outrageous +goldstein +##eak +##kt +ld +maitland +penultimate +pollard +##dance +fleets +spaceship +vertebrae +##nig +alcoholism +als +recital +##bham +##ference +##omics +m2 +##bm +trois +##tropical +##в +commemorates +##meric +marge +##raction +1643 +670 +cosmetic +ravaged +##ige +catastrophe +eng +##shida +albrecht +arterial +bellamy +decor +harmon +##rde +bulbs +synchronized +vito +easiest +shetland +shielding +wnba +##glers +##ssar +##riam +brianna +cumbria +##aceous +##rard +cores +thayer +##nsk +brood +hilltop +luminous +carts +keynote +larkin +logos +##cta +##ا +##mund +##quay +lilith +tinted +277 +wrestle +mobilization +##uses +sequential +siam +bloomfield +takahashi +274 +##ieving +presenters +ringo +blazed +witty +##oven +##ignant +devastation +haydn +harmed +newt +therese +##peed +gershwin +molina +rabbis +sudanese +001 +innate +restarted +##sack +##fus +slices +wb +##shah +enroll +hypothetical +hysterical +1743 +fabio +indefinite +warped +##hg +exchanging +525 +unsuitable +##sboro +gallo +1603 +bret +cobalt +homemade +##hunter +mx +operatives +##dhar +terraces +durable +latch +pens +whorls +##ctuated +##eaux +billing +ligament +succumbed +##gly +regulators +spawn +##brick +##stead +filmfare +rochelle +##nzo +1725 +circumstance +saber +supplements +##nsky +##tson +crowe +wellesley +carrot +##9th +##movable +primate +drury +sincerely +topical +##mad +##rao +callahan +kyiv +smarter +tits +undo +##yeh +announcements +anthologies +barrio +nebula +##islaus +##shaft +##tyn +bodyguards +2021 +assassinate +barns +emmett +scully +##mah +##yd +##eland +##tino +##itarian +demoted +gorman +lashed +prized +adventist +writ +##gui +alla +invertebrates +##ausen +1641 +amman +1742 +align +healy +redistribution +##gf +##rize +insulation +##drop +adherents +hezbollah +vitro +ferns +yanking +269 +php +registering +uppsala +cheerleading +confines +mischievous +tully +##ross +49th +docked +roam +stipulated +pumpkin +##bry +prompt +##ezer +blindly +shuddering +craftsmen +frail +scented +katharine +scramble +shaggy +sponge +helix +zaragoza +279 +##52 +43rd +backlash +fontaine +seizures +posse +cowan +nonfiction +telenovela +wwii +hammered +undone +##gpur +encircled +irs +##ivation +artefacts +oneself +searing +smallpox +##belle +##osaurus +shandong +breached +upland +blushing +rankin +infinitely +psyche +tolerated +docking +evicted +##col +unmarked +##lving +gnome +lettering +litres +musique +##oint +benevolent +##jal +blackened +##anna +mccall +racers +tingle +##ocene +##orestation +introductions +radically +292 +##hiff +##باد +1610 +1739 +munchen +plead +##nka +condo +scissors +##sight +##tens +apprehension +##cey +##yin +hallmark +watering +formulas +sequels +##llas +aggravated +bae +commencing +##building +enfield +prohibits +marne +vedic +civilized +euclidean +jagger +beforehand +blasts +dumont +##arney +##nem +740 +conversions +hierarchical +rios +simulator +##dya +##lellan +hedges +oleg +thrusts +shadowed +darby +maximize +1744 +gregorian +##nded +##routed +sham +unspecified +##hog +emory +factual +##smo +##tp +fooled +##rger +ortega +wellness +marlon +##oton +##urance +casket +keating +ley +enclave +##ayan +char +influencing +jia +##chenko +412 +ammonia +erebidae +incompatible +violins +cornered +##arat +grooves +astronauts +columbian +rampant +fabrication +kyushu +mahmud +vanish +##dern +mesopotamia +##lete +ict +##rgen +caspian +kenji +pitted +##vered +999 +grimace +roanoke +tchaikovsky +twinned +##analysis +##awan +xinjiang +arias +clemson +kazakh +sizable +1662 +##khand +##vard +plunge +tatum +vittorio +##nden +cholera +##dana +##oper +bracing +indifference +projectile +superliga +##chee +realises +upgrading +299 +porte +retribution +##vies +nk +stil +##resses +ama +bureaucracy +blackberry +bosch +testosterone +collapses +greer +##pathic +ioc +fifties +malls +##erved +bao +baskets +adolescents +siegfried +##osity +##tosis +mantra +detecting +existent +fledgling +##cchi +dissatisfied +gan +telecommunication +mingled +sobbed +6000 +controversies +outdated +taxis +##raus +fright +slams +##lham +##fect +##tten +detectors +fetal +tanned +##uw +fray +goth +olympian +skipping +mandates +scratches +sheng +unspoken +hyundai +tracey +hotspur +restrictive +##buch +americana +mundo +##bari +burroughs +diva +vulcan +##6th +distinctions +thumping +##ngen +mikey +sheds +fide +rescues +springsteen +vested +valuation +##ece +##ely +pinnacle +rake +sylvie +##edo +almond +quivering +##irus +alteration +faltered +##wad +51st +hydra +ticked +##kato +recommends +##dicated +antigua +arjun +stagecoach +wilfred +trickle +pronouns +##pon +aryan +nighttime +##anian +gall +pea +stitch +##hei +leung +milos +##dini +eritrea +nexus +starved +snowfall +kant +parasitic +cot +discus +hana +strikers +appleton +kitchens +##erina +##partisan +##itha +##vius +disclose +metis +##channel +1701 +tesla +##vera +fitch +1735 +blooded +##tila +decimal +##tang +##bai +cyclones +eun +bottled +peas +pensacola +basha +bolivian +crabs +boil +lanterns +partridge +roofed +1645 +necks +##phila +opined +patting +##kla +##lland +chuckles +volta +whereupon +##nche +devout +euroleague +suicidal +##dee +inherently +involuntary +knitting +nasser +##hide +puppets +colourful +courageous +southend +stills +miraculous +hodgson +richer +rochdale +ethernet +greta +uniting +prism +umm +##haya +##itical +##utation +deterioration +pointe +prowess +##ropriation +lids +scranton +billings +subcontinent +##koff +##scope +brute +kellogg +psalms +degraded +##vez +stanisław +##ructured +ferreira +pun +astonishing +gunnar +##yat +arya +prc +gottfried +##tight +excursion +##ographer +dina +##quil +##nare +huffington +illustrious +wilbur +gundam +verandah +##zard +naacp +##odle +constructive +fjord +kade +##naud +generosity +thrilling +baseline +cayman +frankish +plastics +accommodations +zoological +##fting +cedric +qb +motorized +##dome +##otted +squealed +tackled +canucks +budgets +situ +asthma +dail +gabled +grasslands +whimpered +writhing +judgments +##65 +minnie +pv +##carbon +bananas +grille +domes +monique +odin +maguire +markham +tierney +##estra +##chua +libel +poke +speedy +atrium +laval +notwithstanding +##edly +fai +kala +##sur +robb +##sma +listings +luz +supplementary +tianjin +##acing +enzo +jd +ric +scanner +croats +transcribed +##49 +arden +cv +##hair +##raphy +##lver +##uy +357 +seventies +staggering +alam +horticultural +hs +regression +timbers +blasting +##ounded +montagu +manipulating +##cit +catalytic +1550 +troopers +##meo +condemnation +fitzpatrick +##oire +##roved +inexperienced +1670 +castes +##lative +outing +314 +dubois +flicking +quarrel +ste +learners +1625 +iq +whistled +##class +282 +classify +tariffs +temperament +355 +folly +liszt +##yles +immersed +jordanian +ceasefire +apparel +extras +maru +fished +##bio +harta +stockport +assortment +craftsman +paralysis +transmitters +##cola +blindness +##wk +fatally +proficiency +solemnly +##orno +repairing +amore +groceries +ultraviolet +##chase +schoolhouse +##tua +resurgence +nailed +##otype +##× +ruse +saliva +diagrams +##tructing +albans +rann +thirties +1b +antennas +hilarious +cougars +paddington +stats +##eger +breakaway +ipod +reza +authorship +prohibiting +scoffed +##etz +##ttle +conscription +defected +trondheim +##fires +ivanov +keenan +##adan +##ciful +##fb +##slow +locating +##ials +##tford +cadiz +basalt +blankly +interned +rags +rattling +##tick +carpathian +reassured +sync +bum +guildford +iss +staunch +##onga +astronomers +sera +sofie +emergencies +susquehanna +##heard +duc +mastery +vh1 +williamsburg +bayer +buckled +craving +##khan +##rdes +bloomington +##write +alton +barbecue +##bians +justine +##hri +##ndt +delightful +smartphone +newtown +photon +retrieval +peugeot +hissing +##monium +##orough +flavors +lighted +relaunched +tainted +##games +##lysis +anarchy +microscopic +hopping +adept +evade +evie +##beau +inhibit +sinn +adjustable +hurst +intuition +wilton +cisco +44th +lawful +lowlands +stockings +thierry +##dalen +##hila +##nai +fates +prank +tb +maison +lobbied +provocative +1724 +4a +utopia +##qual +carbonate +gujarati +purcell +##rford +curtiss +##mei +overgrown +arenas +mediation +swallows +##rnik +respectful +turnbull +##hedron +##hope +alyssa +ozone +##ʻi +ami +gestapo +johansson +snooker +canteen +cuff +declines +empathy +stigma +##ags +##iner +##raine +taxpayers +gui +volga +##wright +##copic +lifespan +overcame +tattooed +enactment +giggles +##ador +##camp +barrington +bribe +obligatory +orbiting +peng +##enas +elusive +sucker +##vating +cong +hardship +empowered +anticipating +estrada +cryptic +greasy +detainees +planck +sudbury +plaid +dod +marriott +kayla +##ears +##vb +##zd +mortally +##hein +cognition +radha +319 +liechtenstein +meade +richly +argyle +harpsichord +liberalism +trumpets +lauded +tyrant +salsa +tiled +lear +promoters +reused +slicing +trident +##chuk +##gami +##lka +cantor +checkpoint +##points +gaul +leger +mammalian +##tov +##aar +##schaft +doha +frenchman +nirvana +##vino +delgado +headlining +##eron +##iography +jug +tko +1649 +naga +intersections +##jia +benfica +nawab +##suka +ashford +gulp +##deck +##vill +##rug +brentford +frazier +pleasures +dunne +potsdam +shenzhen +dentistry +##tec +flanagan +##dorff +##hear +chorale +dinah +prem +quezon +##rogated +relinquished +sutra +terri +##pani +flaps +##rissa +poly +##rnet +homme +aback +##eki +linger +womb +##kson +##lewood +doorstep +orthodoxy +threaded +westfield +##rval +dioceses +fridays +subsided +##gata +loyalists +##biotic +##ettes +letterman +lunatic +prelate +tenderly +invariably +souza +thug +winslow +##otide +furlongs +gogh +jeopardy +##runa +pegasus +##umble +humiliated +standalone +tagged +##roller +freshmen +klan +##bright +attaining +initiating +transatlantic +logged +viz +##uance +1723 +combatants +intervening +stephane +chieftain +despised +grazed +317 +cdc +galveston +godzilla +macro +simulate +##planes +parades +##esses +960 +##ductive +##unes +equator +overdose +##cans +##hosh +##lifting +joshi +epstein +sonora +treacherous +aquatics +manchu +responsive +##sation +supervisory +##christ +##llins +##ibar +##balance +##uso +kimball +karlsruhe +mab +##emy +ignores +phonetic +reuters +spaghetti +820 +almighty +danzig +rumbling +tombstone +designations +lured +outset +##felt +supermarkets +##wt +grupo +kei +kraft +susanna +##blood +comprehension +genealogy +##aghan +##verted +redding +##ythe +1722 +bowing +##pore +##roi +lest +sharpened +fulbright +valkyrie +sikhs +##unds +swans +bouquet +merritt +##tage +##venting +commuted +redhead +clerks +leasing +cesare +dea +hazy +##vances +fledged +greenfield +servicemen +##gical +armando +blackout +dt +sagged +downloadable +intra +potion +pods +##4th +##mism +xp +attendants +gambia +stale +##ntine +plump +asteroids +rediscovered +buds +flea +hive +##neas +1737 +classifications +debuts +##eles +olympus +scala +##eurs +##gno +##mute +hummed +sigismund +visuals +wiggled +await +pilasters +clench +sulfate +##ances +bellevue +enigma +trainee +snort +##sw +clouded +denim +##rank +##rder +churning +hartman +lodges +riches +sima +##missible +accountable +socrates +regulates +mueller +##cr +1702 +avoids +solids +himalayas +nutrient +pup +##jevic +squat +fades +nec +##lates +##pina +##rona +##ου +privateer +tequila +##gative +##mpton +apt +hornet +immortals +##dou +asturias +cleansing +dario +##rries +##anta +etymology +servicing +zhejiang +##venor +##nx +horned +erasmus +rayon +relocating +£10 +##bags +escalated +promenade +stubble +2010s +artisans +axial +liquids +mora +sho +yoo +##tsky +bundles +oldies +##nally +notification +bastion +##ths +sparkle +##lved +1728 +leash +pathogen +highs +##hmi +immature +880 +gonzaga +ignatius +mansions +monterrey +sweets +bryson +##loe +polled +regatta +brightest +pei +rosy +squid +hatfield +payroll +addict +meath +cornerback +heaviest +lodging +##mage +capcom +rippled +##sily +barnet +mayhem +ymca +snuggled +rousseau +##cute +blanchard +284 +fragmented +leighton +chromosomes +risking +##md +##strel +##utter +corinne +coyotes +cynical +hiroshi +yeomanry +##ractive +ebook +grading +mandela +plume +agustin +magdalene +##rkin +bea +femme +trafford +##coll +##lun +##tance +52nd +fourier +upton +##mental +camilla +gust +iihf +islamabad +longevity +##kala +feldman +netting +##rization +endeavour +foraging +mfa +orr +##open +greyish +contradiction +graz +##ruff +handicapped +marlene +tweed +oaxaca +spp +campos +miocene +pri +configured +cooks +pluto +cozy +pornographic +##entes +70th +fairness +glided +jonny +lynne +rounding +sired +##emon +##nist +remade +uncover +##mack +complied +lei +newsweek +##jured +##parts +##enting +##pg +293 +finer +guerrillas +athenian +deng +disused +stepmother +accuse +gingerly +seduction +521 +confronting +##walker +##going +gora +nostalgia +sabres +virginity +wrenched +##minated +syndication +wielding +eyre +##56 +##gnon +##igny +behaved +taxpayer +sweeps +##growth +childless +gallant +##ywood +amplified +geraldine +scrape +##ffi +babylonian +fresco +##rdan +##kney +##position +1718 +restricting +tack +fukuoka +osborn +selector +partnering +##dlow +318 +gnu +kia +tak +whitley +gables +##54 +##mania +mri +softness +immersion +##bots +##evsky +1713 +chilling +insignificant +pcs +##uis +elites +lina +purported +supplemental +teaming +##americana +##dding +##inton +proficient +rouen +##nage +##rret +niccolo +selects +##bread +fluffy +1621 +gruff +knotted +mukherjee +polgara +thrash +nicholls +secluded +smoothing +thru +corsica +loaf +whitaker +inquiries +##rrier +##kam +indochina +289 +marlins +myles +peking +##tea +extracts +pastry +superhuman +connacht +vogel +##ditional +##het +##udged +##lash +gloss +quarries +refit +teaser +##alic +##gaon +20s +materialized +sling +camped +pickering +tung +tracker +pursuant +##cide +cranes +soc +##cini +##typical +##viere +anhalt +overboard +workout +chores +fares +orphaned +stains +##logie +fenton +surpassing +joyah +triggers +##itte +grandmaster +##lass +##lists +clapping +fraudulent +ledger +nagasaki +##cor +##nosis +##tsa +eucalyptus +tun +##icio +##rney +##tara +dax +heroism +ina +wrexham +onboard +unsigned +##dates +moshe +galley +winnie +droplets +exiles +praises +watered +noodles +##aia +fein +adi +leland +multicultural +stink +bingo +comets +erskine +modernized +canned +constraint +domestically +chemotherapy +featherweight +stifled +##mum +darkly +irresistible +refreshing +hasty +isolate +##oys +kitchener +planners +##wehr +cages +yarn +implant +toulon +elects +childbirth +yue +##lind +##lone +cn +rightful +sportsman +junctions +remodeled +specifies +##rgh +291 +##oons +complimented +##urgent +lister +ot +##logic +bequeathed +cheekbones +fontana +gabby +##dial +amadeus +corrugated +maverick +resented +triangles +##hered +##usly +nazareth +tyrol +1675 +assent +poorer +sectional +aegean +##cous +296 +nylon +ghanaian +##egorical +##weig +cushions +forbid +fusiliers +obstruction +somerville +##scia +dime +earrings +elliptical +leyte +oder +polymers +timmy +atm +midtown +piloted +settles +continual +externally +mayfield +##uh +enrichment +henson +keane +persians +1733 +benji +braden +pep +324 +##efe +contenders +pepsi +valet +##isches +298 +##asse +##earing +goofy +stroll +##amen +authoritarian +occurrences +adversary +ahmedabad +tangent +toppled +dorchester +1672 +modernism +marxism +islamist +charlemagne +exponential +racks +unicode +brunette +mbc +pic +skirmish +##bund +##lad +##powered +##yst +hoisted +messina +shatter +##ctum +jedi +vantage +##music +##neil +clemens +mahmoud +corrupted +authentication +lowry +nils +##washed +omnibus +wounding +jillian +##itors +##opped +serialized +narcotics +handheld +##arm +##plicity +intersecting +stimulating +##onis +crate +fellowships +hemingway +casinos +climatic +fordham +copeland +drip +beatty +leaflets +robber +brothel +madeira +##hedral +sphinx +ultrasound +##vana +valor +forbade +leonid +villas +##aldo +duane +marquez +##cytes +disadvantaged +forearms +kawasaki +reacts +consular +lax +uncles +uphold +##hopper +concepcion +dorsey +lass +##izan +arching +passageway +1708 +researches +tia +internationals +##graphs +##opers +distinguishes +javanese +divert +##uven +plotted +##listic +##rwin +##erik +##tify +affirmative +signifies +validation +##bson +kari +felicity +georgina +zulu +##eros +##rained +##rath +overcoming +##dot +argyll +##rbin +1734 +chiba +ratification +windy +earls +parapet +##marks +hunan +pristine +astrid +punta +##gart +brodie +##kota +##oder +malaga +minerva +rouse +##phonic +bellowed +pagoda +portals +reclamation +##gur +##odies +##⁄₄ +parentheses +quoting +allergic +palette +showcases +benefactor +heartland +nonlinear +##tness +bladed +cheerfully +scans +##ety +##hone +1666 +girlfriends +pedersen +hiram +sous +##liche +##nator +1683 +##nery +##orio +##umen +bobo +primaries +smiley +##cb +unearthed +uniformly +fis +metadata +1635 +ind +##oted +recoil +##titles +##tura +##ια +406 +hilbert +jamestown +mcmillan +tulane +seychelles +##frid +antics +coli +fated +stucco +##grants +1654 +bulky +accolades +arrays +caledonian +carnage +optimism +puebla +##tative +##cave +enforcing +rotherham +seo +dunlop +aeronautics +chimed +incline +zoning +archduke +hellenistic +##oses +##sions +candi +thong +##ople +magnate +rustic +##rsk +projective +slant +##offs +danes +hollis +vocalists +##ammed +congenital +contend +gesellschaft +##ocating +##pressive +douglass +quieter +##cm +##kshi +howled +salim +spontaneously +townsville +buena +southport +##bold +kato +1638 +faerie +stiffly +##vus +##rled +297 +flawless +realising +taboo +##7th +bytes +straightening +356 +jena +##hid +##rmin +cartwright +berber +bertram +soloists +411 +noses +417 +coping +fission +hardin +inca +##cen +1717 +mobilized +vhf +##raf +biscuits +curate +##85 +##anial +331 +gaunt +neighbourhoods +1540 +##abas +blanca +bypassed +sockets +behold +coincidentally +##bane +nara +shave +splinter +terrific +##arion +##erian +commonplace +juris +redwood +waistband +boxed +caitlin +fingerprints +jennie +naturalized +##ired +balfour +craters +jody +bungalow +hugely +quilt +glitter +pigeons +undertaker +bulging +constrained +goo +##sil +##akh +assimilation +reworked +##person +persuasion +##pants +felicia +##cliff +##ulent +1732 +explodes +##dun +##inium +##zic +lyman +vulture +hog +overlook +begs +northwards +ow +spoil +##urer +fatima +favorably +accumulate +sargent +sorority +corresponded +dispersal +kochi +toned +##imi +##lita +internacional +newfound +##agger +##lynn +##rigue +booths +peanuts +##eborg +medicare +muriel +nur +##uram +crates +millennia +pajamas +worsened +##breakers +jimi +vanuatu +yawned +##udeau +carousel +##hony +hurdle +##ccus +##mounted +##pod +rv +##eche +airship +ambiguity +compulsion +recapture +##claiming +arthritis +##osomal +1667 +asserting +ngc +sniffing +dade +discontent +glendale +ported +##amina +defamation +rammed +##scent +fling +livingstone +##fleet +875 +##ppy +apocalyptic +comrade +lcd +##lowe +cessna +eine +persecuted +subsistence +demi +hoop +reliefs +710 +coptic +progressing +stemmed +perpetrators +1665 +priestess +##nio +dobson +ebony +rooster +itf +tortricidae +##bbon +##jian +cleanup +##jean +##øy +1721 +eighties +taxonomic +holiness +##hearted +##spar +antilles +showcasing +stabilized +##nb +gia +mascara +michelangelo +dawned +##uria +##vinsky +extinguished +fitz +grotesque +£100 +##fera +##loid +##mous +barges +neue +throbbed +cipher +johnnie +##a1 +##mpt +outburst +##swick +spearheaded +administrations +c1 +heartbreak +pixels +pleasantly +##enay +lombardy +plush +##nsed +bobbie +##hly +reapers +tremor +xiang +minogue +substantive +hitch +barak +##wyl +kwan +##encia +910 +obscene +elegance +indus +surfer +bribery +conserve +##hyllum +##masters +horatio +##fat +apes +rebound +psychotic +##pour +iteration +##mium +##vani +botanic +horribly +antiques +dispose +paxton +##hli +##wg +timeless +1704 +disregard +engraver +hounds +##bau +##version +looted +uno +facilitates +groans +masjid +rutland +antibody +disqualification +decatur +footballers +quake +slacks +48th +rein +scribe +stabilize +commits +exemplary +tho +##hort +##chison +pantry +traversed +##hiti +disrepair +identifiable +vibrated +baccalaureate +##nnis +csa +interviewing +##iensis +##raße +greaves +wealthiest +343 +classed +jogged +£5 +##58 +##atal +illuminating +knicks +respecting +##uno +scrubbed +##iji +##dles +kruger +moods +growls +raider +silvia +chefs +kam +vr +cree +percival +##terol +gunter +counterattack +defiant +henan +ze +##rasia +##riety +equivalence +submissions +##fra +##thor +bautista +mechanically +##heater +cornice +herbal +templar +##mering +outputs +ruining +ligand +renumbered +extravagant +mika +blockbuster +eta +insurrection +##ilia +darkening +ferocious +pianos +strife +kinship +##aer +melee +##anor +##iste +##may +##oue +decidedly +weep +##jad +##missive +##ppel +354 +puget +unease +##gnant +1629 +hammering +kassel +ob +wessex +##lga +bromwich +egan +paranoia +utilization +##atable +##idad +contradictory +provoke +##ols +##ouring +##tangled +knesset +##very +##lette +plumbing +##sden +##¹ +greensboro +occult +sniff +338 +zev +beaming +gamer +haggard +mahal +##olt +##pins +mendes +utmost +briefing +gunnery +##gut +##pher +##zh +##rok +1679 +khalifa +sonya +##boot +principals +urbana +wiring +##liffe +##minating +##rrado +dahl +nyu +skepticism +np +townspeople +ithaca +lobster +somethin +##fur +##arina +##−1 +freighter +zimmerman +biceps +contractual +##herton +amend +hurrying +subconscious +##anal +336 +meng +clermont +spawning +##eia +##lub +dignitaries +impetus +snacks +spotting +twigs +##bilis +##cz +##ouk +libertadores +nic +skylar +##aina +##firm +gustave +asean +##anum +dieter +legislatures +flirt +bromley +trolls +umar +##bbies +##tyle +blah +parc +bridgeport +crank +negligence +##nction +46th +constantin +molded +bandages +seriousness +00pm +siegel +carpets +compartments +upbeat +statehood +##dner +##edging +marko +730 +platt +##hane +paving +##iy +1738 +abbess +impatience +limousine +nbl +##talk +441 +lucille +mojo +nightfall +robbers +##nais +karel +brisk +calves +replicate +ascribed +telescopes +##olf +intimidated +##reen +ballast +specialization +##sit +aerodynamic +caliphate +rainer +visionary +##arded +epsilon +##aday +##onte +aggregation +auditory +boosted +reunification +kathmandu +loco +robyn +402 +acknowledges +appointing +humanoid +newell +redeveloped +restraints +##tained +barbarians +chopper +1609 +italiana +##lez +##lho +investigates +wrestlemania +##anies +##bib +690 +##falls +creaked +dragoons +gravely +minions +stupidity +volley +##harat +##week +musik +##eries +##uously +fungal +massimo +semantics +malvern +##ahl +##pee +discourage +embryo +imperialism +1910s +profoundly +##ddled +jiangsu +sparkled +stat +##holz +sweatshirt +tobin +##iction +sneered +##cheon +##oit +brit +causal +smyth +##neuve +diffuse +perrin +silvio +##ipes +##recht +detonated +iqbal +selma +##nism +##zumi +roasted +##riders +tay +##ados +##mament +##mut +##rud +840 +completes +nipples +cfa +flavour +hirsch +##laus +calderon +sneakers +moravian +##ksha +1622 +rq +294 +##imeters +bodo +##isance +##pre +##ronia +anatomical +excerpt +##lke +dh +kunst +##tablished +##scoe +biomass +panted +unharmed +gael +housemates +montpellier +##59 +coa +rodents +tonic +hickory +singleton +##taro +451 +1719 +aldo +breaststroke +dempsey +och +rocco +##cuit +merton +dissemination +midsummer +serials +##idi +haji +polynomials +##rdon +gs +enoch +prematurely +shutter +taunton +£3 +##grating +##inates +archangel +harassed +##asco +326 +archway +dazzling +##ecin +1736 +sumo +wat +##kovich +1086 +honneur +##ently +##nostic +##ttal +##idon +1605 +403 +1716 +blogger +rents +##gnan +hires +##ikh +##dant +howie +##rons +handler +retracted +shocks +1632 +arun +duluth +kepler +trumpeter +##lary +peeking +seasoned +trooper +##mara +laszlo +##iciencies +##rti +heterosexual +##inatory +##ssion +indira +jogging +##inga +##lism +beit +dissatisfaction +malice +##ately +nedra +peeling +##rgeon +47th +stadiums +475 +vertigo +##ains +iced +restroom +##plify +##tub +illustrating +pear +##chner +##sibility +inorganic +rappers +receipts +watery +##kura +lucinda +##oulos +reintroduced +##8th +##tched +gracefully +saxons +nutritional +wastewater +rained +favourites +bedrock +fisted +hallways +likeness +upscale +##lateral +1580 +blinds +prequel +##pps +##tama +deter +humiliating +restraining +tn +vents +1659 +laundering +recess +rosary +tractors +coulter +federer +##ifiers +##plin +persistence +##quitable +geschichte +pendulum +quakers +##beam +bassett +pictorial +buffet +koln +##sitor +drills +reciprocal +shooters +##57 +##cton +##tees +converge +pip +dmitri +donnelly +yamamoto +aqua +azores +demographics +hypnotic +spitfire +suspend +wryly +roderick +##rran +sebastien +##asurable +mavericks +##fles +##200 +himalayan +prodigy +##iance +transvaal +demonstrators +handcuffs +dodged +mcnamara +sublime +1726 +crazed +##efined +##till +ivo +pondered +reconciled +shrill +sava +##duk +bal +cad +heresy +jaipur +goran +##nished +341 +lux +shelly +whitehall +##hre +israelis +peacekeeping +##wled +1703 +demetrius +ousted +##arians +##zos +beale +anwar +backstroke +raged +shrinking +cremated +##yck +benign +towing +wadi +darmstadt +landfill +parana +soothe +colleen +sidewalks +mayfair +tumble +hepatitis +ferrer +superstructure +##gingly +##urse +##wee +anthropological +translators +##mies +closeness +hooves +##pw +mondays +##roll +##vita +landscaping +##urized +purification +sock +thorns +thwarted +jalan +tiberius +##taka +saline +##rito +confidently +khyber +sculptors +##ij +brahms +hammersmith +inspectors +battista +fivb +fragmentation +hackney +##uls +arresting +exercising +antoinette +bedfordshire +##zily +dyed +##hema +1656 +racetrack +variability +##tique +1655 +austrians +deteriorating +madman +theorists +aix +lehman +weathered +1731 +decreed +eruptions +1729 +flaw +quinlan +sorbonne +flutes +nunez +1711 +adored +downwards +fable +rasped +1712 +moritz +mouthful +renegade +shivers +stunts +dysfunction +restrain +translit +327 +pancakes +##avio +##cision +##tray +351 +vial +##lden +bain +##maid +##oxide +chihuahua +malacca +vimes +##rba +##rnier +1664 +donnie +plaques +##ually +337 +bangs +floppy +huntsville +loretta +nikolay +##otte +eater +handgun +ubiquitous +##hett +eras +zodiac +1634 +##omorphic +1820s +##zog +cochran +##bula +##lithic +warring +##rada +dalai +excused +blazers +mcconnell +reeling +bot +este +##abi +geese +hoax +taxon +##bla +guitarists +##icon +condemning +hunts +inversion +moffat +taekwondo +##lvis +1624 +stammered +##rest +##rzy +sousa +fundraiser +marylebone +navigable +uptown +cabbage +daniela +salman +shitty +whimper +##kian +##utive +programmers +protections +rm +##rmi +##rued +forceful +##enes +fuss +##tao +##wash +brat +oppressive +reykjavik +spartak +ticking +##inkles +##kiewicz +adolph +horst +maui +protege +straighten +cpc +landau +concourse +clements +resultant +##ando +imaginative +joo +reactivated +##rem +##ffled +##uising +consultative +##guide +flop +kaitlyn +mergers +parenting +somber +##vron +supervise +vidhan +##imum +courtship +exemplified +harmonies +medallist +refining +##rrow +##ка +amara +##hum +780 +goalscorer +sited +overshadowed +rohan +displeasure +secretive +multiplied +osman +##orth +engravings +padre +##kali +##veda +miniatures +mis +##yala +clap +pali +rook +##cana +1692 +57th +antennae +astro +oskar +1628 +bulldog +crotch +hackett +yucatan +##sure +amplifiers +brno +ferrara +migrating +##gree +thanking +turing +##eza +mccann +ting +andersson +onslaught +gaines +ganga +incense +standardization +##mation +sentai +scuba +stuffing +turquoise +waivers +alloys +##vitt +regaining +vaults +##clops +##gizing +digger +furry +memorabilia +probing +##iad +payton +rec +deutschland +filippo +opaque +seamen +zenith +afrikaans +##filtration +disciplined +inspirational +##merie +banco +confuse +grafton +tod +##dgets +championed +simi +anomaly +biplane +##ceptive +electrode +##para +1697 +cleavage +crossbow +swirl +informant +##lars +##osta +afi +bonfire +spec +##oux +lakeside +slump +##culus +##lais +##qvist +##rrigan +1016 +facades +borg +inwardly +cervical +xl +pointedly +050 +stabilization +##odon +chests +1699 +hacked +ctv +orthogonal +suzy +##lastic +gaulle +jacobite +rearview +##cam +##erted +ashby +##drik +##igate +##mise +##zbek +affectionately +canine +disperse +latham +##istles +##ivar +spielberg +##orin +##idium +ezekiel +cid +##sg +durga +middletown +##cina +customized +frontiers +harden +##etano +##zzy +1604 +bolsheviks +##66 +coloration +yoko +##bedo +briefs +slabs +debra +liquidation +plumage +##oin +blossoms +dementia +subsidy +1611 +proctor +relational +jerseys +parochial +ter +##ici +esa +peshawar +cavalier +loren +cpi +idiots +shamrock +1646 +dutton +malabar +mustache +##endez +##ocytes +referencing +terminates +marche +yarmouth +##sop +acton +mated +seton +subtly +baptised +beige +extremes +jolted +kristina +telecast +##actic +safeguard +waldo +##baldi +##bular +endeavors +sloppy +subterranean +##ensburg +##itung +delicately +pigment +tq +##scu +1626 +##ound +collisions +coveted +herds +##personal +##meister +##nberger +chopra +##ricting +abnormalities +defective +galician +lucie +##dilly +alligator +likened +##genase +burundi +clears +complexion +derelict +deafening +diablo +fingered +champaign +dogg +enlist +isotope +labeling +mrna +##erre +brilliance +marvelous +##ayo +1652 +crawley +ether +footed +dwellers +deserts +hamish +rubs +warlock +skimmed +##lizer +870 +buick +embark +heraldic +irregularities +##ajan +kiara +##kulam +##ieg +antigen +kowalski +##lge +oakley +visitation +##mbit +vt +##suit +1570 +murderers +##miento +##rites +chimneys +##sling +condemn +custer +exchequer +havre +##ghi +fluctuations +##rations +dfb +hendricks +vaccines +##tarian +nietzsche +biking +juicy +##duced +brooding +scrolling +selangor +##ragan +352 +annum +boomed +seminole +sugarcane +##dna +departmental +dismissing +innsbruck +arteries +ashok +batavia +daze +kun +overtook +##rga +##tlan +beheaded +gaddafi +holm +electronically +faulty +galilee +fractures +kobayashi +##lized +gunmen +magma +aramaic +mala +eastenders +inference +messengers +bf +##qu +407 +bathrooms +##vere +1658 +flashbacks +ideally +misunderstood +##jali +##weather +mendez +##grounds +505 +uncanny +##iii +1709 +friendships +##nbc +sacrament +accommodated +reiterated +logistical +pebbles +thumped +##escence +administering +decrees +drafts +##flight +##cased +##tula +futuristic +picket +intimidation +winthrop +##fahan +interfered +339 +afar +francoise +morally +uta +cochin +croft +dwarfs +##bruck +##dents +##nami +biker +##hner +##meral +nano +##isen +##ometric +##pres +##ан +brightened +meek +parcels +securely +gunners +##jhl +##zko +agile +hysteria +##lten +##rcus +bukit +champs +chevy +cuckoo +leith +sadler +theologians +welded +##section +1663 +jj +plurality +xander +##rooms +##formed +shredded +temps +intimately +pau +tormented +##lok +##stellar +1618 +charred +ems +essen +##mmel +alarms +spraying +ascot +blooms +twinkle +##abia +##apes +internment +obsidian +##chaft +snoop +##dav +##ooping +malibu +##tension +quiver +##itia +hays +mcintosh +travers +walsall +##ffie +1623 +beverley +schwarz +plunging +structurally +m3 +rosenthal +vikram +##tsk +770 +ghz +##onda +##tiv +chalmers +groningen +pew +reckon +unicef +##rvis +55th +##gni +1651 +sulawesi +avila +cai +metaphysical +screwing +turbulence +##mberg +augusto +samba +56th +baffled +momentary +toxin +##urian +##wani +aachen +condoms +dali +steppe +##3d +##app +##oed +##year +adolescence +dauphin +electrically +inaccessible +microscopy +nikita +##ega +atv +##cel +##enter +##oles +##oteric +##ы +accountants +punishments +wrongly +bribes +adventurous +clinch +flinders +southland +##hem +##kata +gough +##ciency +lads +soared +##ה +undergoes +deformation +outlawed +rubbish +##arus +##mussen +##nidae +##rzburg +arcs +##ingdon +##tituted +1695 +wheelbase +wheeling +bombardier +campground +zebra +##lices +##oj +##bain +lullaby +##ecure +donetsk +wylie +grenada +##arding +##ης +squinting +eireann +opposes +##andra +maximal +runes +##broken +##cuting +##iface +##ror +##rosis +additive +britney +adultery +triggering +##drome +detrimental +aarhus +containment +jc +swapped +vichy +##ioms +madly +##oric +##rag +brant +##ckey +##trix +1560 +1612 +broughton +rustling +##stems +##uder +asbestos +mentoring +##nivorous +finley +leaps +##isan +apical +pry +slits +substitutes +##dict +intuitive +fantasia +insistent +unreasonable +##igen +##vna +domed +hannover +margot +ponder +##zziness +impromptu +jian +lc +rampage +stemming +##eft +andrey +gerais +whichever +amnesia +appropriated +anzac +clicks +modifying +ultimatum +cambrian +maids +verve +yellowstone +##mbs +conservatoire +##scribe +adherence +dinners +spectra +imperfect +mysteriously +sidekick +tatar +tuba +##aks +##ifolia +distrust +##athan +##zle +c2 +ronin +zac +##pse +celaena +instrumentalist +scents +skopje +##mbling +comical +compensated +vidal +condor +intersect +jingle +wavelengths +##urrent +mcqueen +##izzly +carp +weasel +422 +kanye +militias +postdoctoral +eugen +gunslinger +##ɛ +faux +hospice +##for +appalled +derivation +dwarves +##elis +dilapidated +##folk +astoria +philology +##lwyn +##otho +##saka +inducing +philanthropy +##bf +##itative +geek +markedly +sql +##yce +bessie +indices +rn +##flict +495 +frowns +resolving +weightlifting +tugs +cleric +contentious +1653 +mania +rms +##miya +##reate +##ruck +##tucket +bien +eels +marek +##ayton +##cence +discreet +unofficially +##ife +leaks +##bber +1705 +332 +dung +compressor +hillsborough +pandit +shillings +distal +##skin +381 +##tat +##you +nosed +##nir +mangrove +undeveloped +##idia +textures +##inho +##500 +##rise +ae +irritating +nay +amazingly +bancroft +apologetic +compassionate +kata +symphonies +##lovic +airspace +##lch +930 +gifford +precautions +fulfillment +sevilla +vulgar +martinique +##urities +looting +piccolo +tidy +##dermott +quadrant +armchair +incomes +mathematicians +stampede +nilsson +##inking +##scan +foo +quarterfinal +##ostal +shang +shouldered +squirrels +##owe +344 +vinegar +##bner +##rchy +##systems +delaying +##trics +ars +dwyer +rhapsody +sponsoring +##gration +bipolar +cinder +starters +##olio +##urst +421 +signage +##nty +aground +figurative +mons +acquaintances +duets +erroneously +soyuz +elliptic +recreated +##cultural +##quette +##ssed +##tma +##zcz +moderator +scares +##itaire +##stones +##udence +juniper +sighting +##just +##nsen +britten +calabria +ry +bop +cramer +forsyth +stillness +##л +airmen +gathers +unfit +##umber +##upt +taunting +##rip +seeker +streamlined +##bution +holster +schumann +tread +vox +##gano +##onzo +strive +dil +reforming +covent +newbury +predicting +##orro +decorate +tre +##puted +andover +ie +asahi +dept +dunkirk +gills +##tori +buren +huskies +##stis +##stov +abstracts +bets +loosen +##opa +1682 +yearning +##glio +##sir +berman +effortlessly +enamel +napoli +persist +##peration +##uez +attache +elisa +b1 +invitations +##kic +accelerating +reindeer +boardwalk +clutches +nelly +polka +starbucks +##kei +adamant +huey +lough +unbroken +adventurer +embroidery +inspecting +stanza +##ducted +naia +taluka +##pone +##roids +chases +deprivation +florian +##jing +##ppet +earthly +##lib +##ssee +colossal +foreigner +vet +freaks +patrice +rosewood +triassic +upstate +##pkins +dominates +ata +chants +ks +vo +##400 +##bley +##raya +##rmed +555 +agra +infiltrate +##ailing +##ilation +##tzer +##uppe +##werk +binoculars +enthusiast +fujian +squeak +##avs +abolitionist +almeida +boredom +hampstead +marsden +rations +##ands +inflated +334 +bonuses +rosalie +patna +##rco +329 +detachments +penitentiary +54th +flourishing +woolf +##dion +##etched +papyrus +##lster +##nsor +##toy +bobbed +dismounted +endelle +inhuman +motorola +tbs +wince +wreath +##ticus +hideout +inspections +sanjay +disgrace +infused +pudding +stalks +##urbed +arsenic +leases +##hyl +##rrard +collarbone +##waite +##wil +dowry +##bant +##edance +genealogical +nitrate +salamanca +scandals +thyroid +necessitated +##! +##" +### +##$ +##% +##& +##' +##( +##) +##* +##+ +##, +##- +##. +##/ +##: +##; +##< +##= +##> +##? +##@ +##[ +##\ +##] +##^ +##_ +##` +##{ +##| +##} +##~ +##¡ +##¢ +##£ +##¤ +##¥ +##¦ +##§ +##¨ +##© +##ª +##« +##¬ +##® +##± +##´ +##µ +##¶ +##· +##º +##» +##¼ +##¾ +##¿ +##æ +##ð +##÷ +##þ +##đ +##ħ +##ŋ +##œ +##ƒ +##ɐ +##ɑ +##ɒ +##ɔ +##ɕ +##ə +##ɡ +##ɣ +##ɨ +##ɪ +##ɫ +##ɬ +##ɯ +##ɲ +##ɴ +##ɹ +##ɾ +##ʀ +##ʁ +##ʂ +##ʃ +##ʉ +##ʊ +##ʋ +##ʌ +##ʎ +##ʐ +##ʑ +##ʒ +##ʔ +##ʰ +##ʲ +##ʳ +##ʷ +##ʸ +##ʻ +##ʼ +##ʾ +##ʿ +##ˈ +##ˡ +##ˢ +##ˣ +##ˤ +##β +##γ +##δ +##ε +##ζ +##θ +##κ +##λ +##μ +##ξ +##ο +##π +##ρ +##σ +##τ +##υ +##φ +##χ +##ψ +##ω +##б +##г +##д +##ж +##з +##м +##п +##с +##у +##ф +##х +##ц +##ч +##ш +##щ +##ъ +##э +##ю +##ђ +##є +##і +##ј +##љ +##њ +##ћ +##ӏ +##ա +##բ +##գ +##դ +##ե +##թ +##ի +##լ +##կ +##հ +##մ +##յ +##ն +##ո +##պ +##ս +##վ +##տ +##ր +##ւ +##ք +##־ +##א +##ב +##ג +##ד +##ו +##ז +##ח +##ט +##י +##ך +##כ +##ל +##ם +##מ +##ן +##נ +##ס +##ע +##ף +##פ +##ץ +##צ +##ק +##ר +##ש +##ת +##، +##ء +##ب +##ت +##ث +##ج +##ح +##خ +##ذ +##ز +##س +##ش +##ص +##ض +##ط +##ظ +##ع +##غ +##ـ +##ف +##ق +##ك +##و +##ى +##ٹ +##پ +##چ +##ک +##گ +##ں +##ھ +##ہ +##ے +##अ +##आ +##उ +##ए +##क +##ख +##ग +##च +##ज +##ट +##ड +##ण +##त +##थ +##द +##ध +##न +##प +##ब +##भ +##म +##य +##र +##ल +##व +##श +##ष +##स +##ह +##ा +##ि +##ी +##ो +##। +##॥ +##ং +##অ +##আ +##ই +##উ +##এ +##ও +##ক +##খ +##গ +##চ +##ছ +##জ +##ট +##ড +##ণ +##ত +##থ +##দ +##ধ +##ন +##প +##ব +##ভ +##ম +##য +##র +##ল +##শ +##ষ +##স +##হ +##া +##ি +##ী +##ে +##க +##ச +##ட +##த +##ந +##ன +##ப +##ம +##ய +##ர +##ல +##ள +##வ +##ா +##ி +##ு +##ே +##ை +##ನ +##ರ +##ಾ +##ක +##ය +##ර +##ල +##ව +##ා +##ก +##ง +##ต +##ท +##น +##พ +##ม +##ย +##ร +##ล +##ว +##ส +##อ +##า +##เ +##་ +##། +##ག +##ང +##ད +##ན +##པ +##བ +##མ +##འ +##ར +##ལ +##ས +##မ +##ა +##ბ +##გ +##დ +##ე +##ვ +##თ +##ი +##კ +##ლ +##მ +##ნ +##ო +##რ +##ს +##ტ +##უ +##ᄀ +##ᄂ +##ᄃ +##ᄅ +##ᄆ +##ᄇ +##ᄉ +##ᄊ +##ᄋ +##ᄌ +##ᄎ +##ᄏ +##ᄐ +##ᄑ +##ᄒ +##ᅡ +##ᅢ +##ᅥ +##ᅦ +##ᅧ +##ᅩ +##ᅪ +##ᅭ +##ᅮ +##ᅯ +##ᅲ +##ᅳ +##ᅴ +##ᅵ +##ᆨ +##ᆫ +##ᆯ +##ᆷ +##ᆸ +##ᆼ +##ᴬ +##ᴮ +##ᴰ +##ᴵ +##ᴺ +##ᵀ +##ᵃ +##ᵇ +##ᵈ +##ᵉ +##ᵍ +##ᵏ +##ᵐ +##ᵒ +##ᵖ +##ᵗ +##ᵘ +##ᵣ +##ᵤ +##ᵥ +##ᶜ +##ᶠ +##‐ +##‑ +##‒ +##– +##— +##― +##‖ +##‘ +##’ +##‚ +##“ +##” +##„ +##† +##‡ +##• +##… +##‰ +##′ +##″ +##› +##‿ +##⁄ +##⁰ +##ⁱ +##⁴ +##⁵ +##⁶ +##⁷ +##⁸ +##⁹ +##⁻ +##ⁿ +##₅ +##₆ +##₇ +##₈ +##₉ +##₊ +##₍ +##₎ +##ₐ +##ₑ +##ₒ +##ₓ +##ₕ +##ₖ +##ₗ +##ₘ +##ₚ +##ₛ +##ₜ +##₤ +##₩ +##€ +##₱ +##₹ +##ℓ +##№ +##ℝ +##™ +##⅓ +##⅔ +##← +##↑ +##→ +##↓ +##↔ +##↦ +##⇄ +##⇌ +##⇒ +##∂ +##∅ +##∆ +##∇ +##∈ +##∗ +##∘ +##√ +##∞ +##∧ +##∨ +##∩ +##∪ +##≈ +##≡ +##≤ +##≥ +##⊂ +##⊆ +##⊕ +##⊗ +##⋅ +##─ +##│ +##■ +##▪ +##● +##★ +##☆ +##☉ +##♠ +##♣ +##♥ +##♦ +##♯ +##⟨ +##⟩ +##ⱼ +##⺩ +##⺼ +##⽥ +##、 +##。 +##〈 +##〉 +##《 +##》 +##「 +##」 +##『 +##』 +##〜 +##あ +##い +##う +##え +##お +##か +##き +##く +##け +##こ +##さ +##し +##す +##せ +##そ +##た +##ち +##っ +##つ +##て +##と +##な +##に +##ぬ +##ね +##の +##は +##ひ +##ふ +##へ +##ほ +##ま +##み +##む +##め +##も +##や +##ゆ +##よ +##ら +##り +##る +##れ +##ろ +##を +##ん +##ァ +##ア +##ィ +##イ +##ウ +##ェ +##エ +##オ +##カ +##キ +##ク +##ケ +##コ +##サ +##シ +##ス +##セ +##タ +##チ +##ッ +##ツ +##テ +##ト +##ナ +##ニ +##ノ +##ハ +##ヒ +##フ +##ヘ +##ホ +##マ +##ミ +##ム +##メ +##モ +##ャ +##ュ +##ョ +##ラ +##リ +##ル +##レ +##ロ +##ワ +##ン +##・ +##ー +##一 +##三 +##上 +##下 +##不 +##世 +##中 +##主 +##久 +##之 +##也 +##事 +##二 +##五 +##井 +##京 +##人 +##亻 +##仁 +##介 +##代 +##仮 +##伊 +##会 +##佐 +##侍 +##保 +##信 +##健 +##元 +##光 +##八 +##公 +##内 +##出 +##分 +##前 +##劉 +##力 +##加 +##勝 +##北 +##区 +##十 +##千 +##南 +##博 +##原 +##口 +##古 +##史 +##司 +##合 +##吉 +##同 +##名 +##和 +##囗 +##四 +##国 +##國 +##土 +##地 +##坂 +##城 +##堂 +##場 +##士 +##夏 +##外 +##大 +##天 +##太 +##夫 +##奈 +##女 +##子 +##学 +##宀 +##宇 +##安 +##宗 +##定 +##宣 +##宮 +##家 +##宿 +##寺 +##將 +##小 +##尚 +##山 +##岡 +##島 +##崎 +##川 +##州 +##巿 +##帝 +##平 +##年 +##幸 +##广 +##弘 +##張 +##彳 +##後 +##御 +##德 +##心 +##忄 +##志 +##忠 +##愛 +##成 +##我 +##戦 +##戸 +##手 +##扌 +##政 +##文 +##新 +##方 +##日 +##明 +##星 +##春 +##昭 +##智 +##曲 +##書 +##月 +##有 +##朝 +##木 +##本 +##李 +##村 +##東 +##松 +##林 +##森 +##楊 +##樹 +##橋 +##歌 +##止 +##正 +##武 +##比 +##氏 +##民 +##水 +##氵 +##氷 +##永 +##江 +##沢 +##河 +##治 +##法 +##海 +##清 +##漢 +##瀬 +##火 +##版 +##犬 +##王 +##生 +##田 +##男 +##疒 +##発 +##白 +##的 +##皇 +##目 +##相 +##省 +##真 +##石 +##示 +##社 +##神 +##福 +##禾 +##秀 +##秋 +##空 +##立 +##章 +##竹 +##糹 +##美 +##義 +##耳 +##良 +##艹 +##花 +##英 +##華 +##葉 +##藤 +##行 +##街 +##西 +##見 +##訁 +##語 +##谷 +##貝 +##貴 +##車 +##軍 +##辶 +##道 +##郎 +##郡 +##部 +##都 +##里 +##野 +##金 +##鈴 +##镇 +##長 +##門 +##間 +##阝 +##阿 +##陳 +##陽 +##雄 +##青 +##面 +##風 +##食 +##香 +##馬 +##高 +##龍 +##龸 +##fi +##fl +##！ +##（ +##） +##， +##－ +##． +##／ +##： +##？ +##～ diff --git a/checkpoints/groundingdino_swinb_cogcoor.pth b/checkpoints/groundingdino_swinb_cogcoor.pth new file mode 100644 index 0000000000000000000000000000000000000000..5f82241224563baebffdc579f20f01a5d9a76c64 --- /dev/null +++ b/checkpoints/groundingdino_swinb_cogcoor.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:46270f7a822e6906b655b729c90613e48929d0f2bb8b9b76fd10a856f3ac6ab7 +size 938057991 diff --git a/datasets/.odvg.py.swp b/datasets/.odvg.py.swp new file mode 100644 index 0000000000000000000000000000000000000000..820e8f2a75eb51a875266419acd553d0cb16ed69 Binary files /dev/null and b/datasets/.odvg.py.swp differ diff --git a/datasets/__init__.py b/datasets/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..b1864e13e91e60944a78de5b7903228cce165cdb --- /dev/null +++ b/datasets/__init__.py @@ -0,0 +1,23 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved +import torch.utils.data +import torchvision +from .coco import build as build_coco + + +def get_coco_api_from_dataset(dataset): + for _ in range(10): + # if isinstance(dataset, torchvision.datasets.CocoDetection): + # break + if isinstance(dataset, torch.utils.data.Subset): + dataset = dataset.dataset + if isinstance(dataset, torchvision.datasets.CocoDetection): + return dataset.coco + + +def build_dataset(image_set, args, datasetinfo): + if datasetinfo["dataset_mode"] == 'coco': + return build_coco(image_set, args, datasetinfo) + if datasetinfo["dataset_mode"] == 'odvg': + from .odvg import build_odvg + return build_odvg(image_set, args, datasetinfo) + raise ValueError(f'dataset {args.dataset_file} not supported') diff --git a/datasets/__pycache__/__init__.cpython-39.pyc b/datasets/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6682a188d531f988e59d3c46a6e651780407e4b6 Binary files /dev/null and b/datasets/__pycache__/__init__.cpython-39.pyc differ diff --git a/datasets/__pycache__/coco.cpython-39.pyc b/datasets/__pycache__/coco.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..08713cfb5abf0f81ffc1c928d1b0ebf2d0af4735 Binary files /dev/null and b/datasets/__pycache__/coco.cpython-39.pyc differ diff --git a/datasets/__pycache__/data_util.cpython-39.pyc b/datasets/__pycache__/data_util.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1c65478f24954f7dbbbaa005c7718b7820966b12 Binary files /dev/null and b/datasets/__pycache__/data_util.cpython-39.pyc differ diff --git a/datasets/__pycache__/transforms.cpython-39.pyc b/datasets/__pycache__/transforms.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..168225c560a1df6775d12533e01f67552256b8d4 Binary files /dev/null and b/datasets/__pycache__/transforms.cpython-39.pyc differ diff --git a/datasets/coco.py b/datasets/coco.py new file mode 100644 index 0000000000000000000000000000000000000000..51cabf8652b2ed42ba6f69e4794d25ceef24445b --- /dev/null +++ b/datasets/coco.py @@ -0,0 +1,827 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved +""" +COCO dataset which returns image_id for evaluation. + +Mostly copy-paste from https://github.com/pytorch/vision/blob/13b35ff/references/detection/coco_utils.py +""" +if __name__ == "__main__": + # for debug only + import os, sys + + sys.path.append(os.path.dirname(sys.path[0])) +from torchvision.datasets.vision import VisionDataset + +import json +from pathlib import Path +import random +import os +from typing import Any, Callable, List, Optional, Tuple + +from PIL import Image + +import torch +import torch.utils.data +import torchvision +from pycocotools import mask as coco_mask + +from datasets.data_util import preparing_dataset +import datasets.transforms as T +from util.box_ops import box_cxcywh_to_xyxy, box_iou + +__all__ = ["build"] + + +class label2compat: + def __init__(self) -> None: + self.category_map_str = { + "1": 1, + "2": 2, + "3": 3, + "4": 4, + "5": 5, + "6": 6, + "7": 7, + "8": 8, + "9": 9, + "10": 10, + "11": 11, + "13": 12, + "14": 13, + "15": 14, + "16": 15, + "17": 16, + "18": 17, + "19": 18, + "20": 19, + "21": 20, + "22": 21, + "23": 22, + "24": 23, + "25": 24, + "27": 25, + "28": 26, + "31": 27, + "32": 28, + "33": 29, + "34": 30, + "35": 31, + "36": 32, + "37": 33, + "38": 34, + "39": 35, + "40": 36, + "41": 37, + "42": 38, + "43": 39, + "44": 40, + "46": 41, + "47": 42, + "48": 43, + "49": 44, + "50": 45, + "51": 46, + "52": 47, + "53": 48, + "54": 49, + "55": 50, + "56": 51, + "57": 52, + "58": 53, + "59": 54, + "60": 55, + "61": 56, + "62": 57, + "63": 58, + "64": 59, + "65": 60, + "67": 61, + "70": 62, + "72": 63, + "73": 64, + "74": 65, + "75": 66, + "76": 67, + "77": 68, + "78": 69, + "79": 70, + "80": 71, + "81": 72, + "82": 73, + "84": 74, + "85": 75, + "86": 76, + "87": 77, + "88": 78, + "89": 79, + "90": 80, + } + self.category_map = {int(k): v for k, v in self.category_map_str.items()} + + def __call__(self, target, img=None): + labels = target["labels"] + res = torch.zeros(labels.shape, dtype=labels.dtype) + for idx, item in enumerate(labels): + res[idx] = self.category_map[item.item()] - 1 + target["label_compat"] = res + if img is not None: + return target, img + else: + return target + + +class label_compat2onehot: + def __init__(self, num_class=80, num_output_objs=1): + self.num_class = num_class + self.num_output_objs = num_output_objs + if num_output_objs != 1: + raise DeprecationWarning( + "num_output_objs!=1, which is only used for comparison" + ) + + def __call__(self, target, img=None): + labels = target["label_compat"] + place_dict = {k: 0 for k in range(self.num_class)} + if self.num_output_objs == 1: + res = torch.zeros(self.num_class) + for i in labels: + itm = i.item() + res[itm] = 1.0 + else: + # compat with baseline + res = torch.zeros(self.num_class, self.num_output_objs) + for i in labels: + itm = i.item() + res[itm][place_dict[itm]] = 1.0 + place_dict[itm] += 1 + target["label_compat_onehot"] = res + if img is not None: + return target, img + else: + return target + + +class box_label_catter: + def __init__(self): + pass + + def __call__(self, target, img=None): + labels = target["label_compat"] + boxes = target["boxes"] + box_label = torch.cat((boxes, labels.unsqueeze(-1)), 1) + target["box_label"] = box_label + if img is not None: + return target, img + else: + return target + + +class RandomSelectBoxlabels: + def __init__( + self, + num_classes, + leave_one_out=False, + blank_prob=0.8, + prob_first_item=0.0, + prob_random_item=0.0, + prob_last_item=0.8, + prob_stop_sign=0.2, + ) -> None: + self.num_classes = num_classes + self.leave_one_out = leave_one_out + self.blank_prob = blank_prob + + self.set_state( + prob_first_item, prob_random_item, prob_last_item, prob_stop_sign + ) + + def get_state(self): + return [ + self.prob_first_item, + self.prob_random_item, + self.prob_last_item, + self.prob_stop_sign, + ] + + def set_state( + self, prob_first_item, prob_random_item, prob_last_item, prob_stop_sign + ): + sum_prob = prob_first_item + prob_random_item + prob_last_item + prob_stop_sign + assert sum_prob - 1 < 1e-6, ( + f"Sum up all prob = {sum_prob}. prob_first_item:{prob_first_item}" + + f"prob_random_item:{prob_random_item}, prob_last_item:{prob_last_item}" + + f"prob_stop_sign:{prob_stop_sign}" + ) + + self.prob_first_item = prob_first_item + self.prob_random_item = prob_random_item + self.prob_last_item = prob_last_item + self.prob_stop_sign = prob_stop_sign + + def sample_for_pred_first_item(self, box_label: torch.FloatTensor): + box_label_known = torch.Tensor(0, 5) + box_label_unknown = box_label + return box_label_known, box_label_unknown + + def sample_for_pred_random_item(self, box_label: torch.FloatTensor): + n_select = int(random.random() * box_label.shape[0]) + box_label = box_label[torch.randperm(box_label.shape[0])] + box_label_known = box_label[:n_select] + box_label_unknown = box_label[n_select:] + return box_label_known, box_label_unknown + + def sample_for_pred_last_item(self, box_label: torch.FloatTensor): + box_label_perm = box_label[torch.randperm(box_label.shape[0])] + known_label_list = [] + box_label_known = [] + box_label_unknown = [] + for item in box_label_perm: + label_i = item[4].item() + if label_i in known_label_list: + box_label_known.append(item) + else: + # first item + box_label_unknown.append(item) + known_label_list.append(label_i) + box_label_known = ( + torch.stack(box_label_known) + if len(box_label_known) > 0 + else torch.Tensor(0, 5) + ) + box_label_unknown = ( + torch.stack(box_label_unknown) + if len(box_label_unknown) > 0 + else torch.Tensor(0, 5) + ) + return box_label_known, box_label_unknown + + def sample_for_pred_stop_sign(self, box_label: torch.FloatTensor): + box_label_unknown = torch.Tensor(0, 5) + box_label_known = box_label + return box_label_known, box_label_unknown + + def __call__(self, target, img=None): + box_label = target["box_label"] # K, 5 + + dice_number = random.random() + + if dice_number < self.prob_first_item: + box_label_known, box_label_unknown = self.sample_for_pred_first_item( + box_label + ) + elif dice_number < self.prob_first_item + self.prob_random_item: + box_label_known, box_label_unknown = self.sample_for_pred_random_item( + box_label + ) + elif ( + dice_number + < self.prob_first_item + self.prob_random_item + self.prob_last_item + ): + box_label_known, box_label_unknown = self.sample_for_pred_last_item( + box_label + ) + else: + box_label_known, box_label_unknown = self.sample_for_pred_stop_sign( + box_label + ) + + target["label_onehot_known"] = label2onehot( + box_label_known[:, -1], self.num_classes + ) + target["label_onehot_unknown"] = label2onehot( + box_label_unknown[:, -1], self.num_classes + ) + target["box_label_known"] = box_label_known + target["box_label_unknown"] = box_label_unknown + + return target, img + + +class RandomDrop: + def __init__(self, p=0.2) -> None: + self.p = p + + def __call__(self, target, img=None): + known_box = target["box_label_known"] + num_known_box = known_box.size(0) + idxs = torch.rand(num_known_box) + # indices = torch.randperm(num_known_box)[:int((1-self).p*num_known_box + 0.5 + random.random())] + target["box_label_known"] = known_box[idxs > self.p] + return target, img + + +class BboxPertuber: + def __init__(self, max_ratio=0.02, generate_samples=1000) -> None: + self.max_ratio = max_ratio + self.generate_samples = generate_samples + self.samples = self.generate_pertube_samples() + self.idx = 0 + + def generate_pertube_samples(self): + import torch + + samples = (torch.rand(self.generate_samples, 5) - 0.5) * 2 * self.max_ratio + return samples + + def __call__(self, target, img): + known_box = target["box_label_known"] # Tensor(K,5), K known bbox + K = known_box.shape[0] + known_box_pertube = torch.zeros(K, 6) # 4:bbox, 1:prob, 1:label + if K == 0: + pass + else: + if self.idx + K > self.generate_samples: + self.idx = 0 + delta = self.samples[self.idx : self.idx + K, :] + known_box_pertube[:, :4] = known_box[:, :4] + delta[:, :4] + iou = ( + torch.diag( + box_iou( + box_cxcywh_to_xyxy(known_box[:, :4]), + box_cxcywh_to_xyxy(known_box_pertube[:, :4]), + )[0] + ) + ) * (1 + delta[:, -1]) + known_box_pertube[:, 4].copy_(iou) + known_box_pertube[:, -1].copy_(known_box[:, -1]) + + target["box_label_known_pertube"] = known_box_pertube + return target, img + + +class RandomCutout: + def __init__(self, factor=0.5) -> None: + self.factor = factor + + def __call__(self, target, img=None): + unknown_box = target["box_label_unknown"] # Ku, 5 + known_box = target["box_label_known_pertube"] # Kk, 6 + Ku = unknown_box.size(0) + + known_box_add = torch.zeros(Ku, 6) # Ku, 6 + known_box_add[:, :5] = unknown_box + known_box_add[:, 5].uniform_(0.5, 1) + + known_box_add[:, :2] += known_box_add[:, 2:4] * (torch.rand(Ku, 2) - 0.5) / 2 + known_box_add[:, 2:4] /= 2 + + target["box_label_known_pertube"] = torch.cat((known_box, known_box_add)) + return target, img + + +class RandomSelectBoxes: + def __init__(self, num_class=80) -> None: + Warning("This is such a slow function and will be deprecated soon!!!") + self.num_class = num_class + + def __call__(self, target, img=None): + boxes = target["boxes"] + labels = target["label_compat"] + + # transform to list of tensors + boxs_list = [[] for i in range(self.num_class)] + for idx, item in enumerate(boxes): + label = labels[idx].item() + boxs_list[label].append(item) + boxs_list_tensor = [ + torch.stack(i) if len(i) > 0 else torch.Tensor(0, 4) for i in boxs_list + ] + + # random selection + box_known = [] + box_unknown = [] + for idx, item in enumerate(boxs_list_tensor): + ncnt = item.shape[0] + nselect = int( + random.random() * ncnt + ) # close in both sides, much faster than random.randint + + item = item[torch.randperm(ncnt)] + # random.shuffle(item) + box_known.append(item[:nselect]) + box_unknown.append(item[nselect:]) + + # box_known_tensor = [torch.stack(i) if len(i) > 0 else torch.Tensor(0,4) for i in box_known] + # box_unknown_tensor = [torch.stack(i) if len(i) > 0 else torch.Tensor(0,4) for i in box_unknown] + # print('box_unknown_tensor:', box_unknown_tensor) + target["known_box"] = box_known + target["unknown_box"] = box_unknown + return target, img + + +def label2onehot(label, num_classes): + """ + label: Tensor(K) + """ + res = torch.zeros(num_classes) + for i in label: + itm = int(i.item()) + res[itm] = 1.0 + return res + + +class MaskCrop: + def __init__(self) -> None: + pass + + def __call__(self, target, img): + known_box = target["known_box"] + h, w = img.shape[1:] # h,w + # imgsize = target['orig_size'] # h,w + + scale = torch.Tensor([w, h, w, h]) + + # _cnt = 0 + for boxes in known_box: + if boxes.shape[0] == 0: + continue + box_xyxy = box_cxcywh_to_xyxy(boxes) * scale + for box in box_xyxy: + x1, y1, x2, y2 = [int(i) for i in box.tolist()] + img[:, y1:y2, x1:x2] = 0 + # _cnt += 1 + # print("_cnt:", _cnt) + return target, img + + +dataset_hook_register = { + "label2compat": label2compat, + "label_compat2onehot": label_compat2onehot, + "box_label_catter": box_label_catter, + "RandomSelectBoxlabels": RandomSelectBoxlabels, + "RandomSelectBoxes": RandomSelectBoxes, + "MaskCrop": MaskCrop, + "BboxPertuber": BboxPertuber, +} + + +class CocoDetection(torchvision.datasets.CocoDetection): + def __init__( + self, img_folder, ann_file, transforms, return_masks, aux_target_hacks=None + ): + super(CocoDetection, self).__init__(img_folder, ann_file) + self._transforms = transforms + self.prepare = ConvertCocoPolysToMask(return_masks) + self.aux_target_hacks = aux_target_hacks + + def change_hack_attr(self, hackclassname, attrkv_dict): + target_class = dataset_hook_register[hackclassname] + for item in self.aux_target_hacks: + if isinstance(item, target_class): + for k, v in attrkv_dict.items(): + setattr(item, k, v) + + def get_hack(self, hackclassname): + target_class = dataset_hook_register[hackclassname] + for item in self.aux_target_hacks: + if isinstance(item, target_class): + return item + + def _load_image(self, id: int) -> Image.Image: + path = self.coco.loadImgs(id)[0]["file_name"] + abs_path = os.path.join(self.root, path) + return Image.open(abs_path).convert("RGB") + + def __getitem__(self, idx): + """ + Output: + - target: dict of multiple items + - boxes: Tensor[num_box, 4]. \ + Init type: x0,y0,x1,y1. unnormalized data. + Final type: cx,cy,w,h. normalized data. + """ + try: + img, target = super(CocoDetection, self).__getitem__(idx) + + except: + print("Error idx: {}".format(idx)) + idx += 1 + img, target = super(CocoDetection, self).__getitem__(idx) + + image_id = self.ids[idx] + target = {"image_id": image_id, "annotations": target} + exemp_count = 0 + for instance in target["annotations"]: + if instance["area"] != 4: + exemp_count += 1 + # Only provide at most 3 visual exemplars during inference. + assert exemp_count == 3 + img, target = self.prepare(img, target) + target["exemplars"] = target["boxes"][-3:] + # Remove inaccurate exemplars. + if image_id == 6003: + target["exemplars"] = torch.tensor([]) + target["boxes"] = target["boxes"][:-3] + target["labels"] = target["labels"][:-3] + target["labels_uncropped"] = torch.clone(target["labels"]) + + if self._transforms is not None: + img, target = self._transforms(img, target) + + # convert to needed format + if self.aux_target_hacks is not None: + for hack_runner in self.aux_target_hacks: + target, img = hack_runner(target, img=img) + + return img, target + + +def convert_coco_poly_to_mask(segmentations, height, width): + masks = [] + for polygons in segmentations: + rles = coco_mask.frPyObjects(polygons, height, width) + mask = coco_mask.decode(rles) + if len(mask.shape) < 3: + mask = mask[..., None] + mask = torch.as_tensor(mask, dtype=torch.uint8) + mask = mask.any(dim=2) + masks.append(mask) + if masks: + masks = torch.stack(masks, dim=0) + else: + masks = torch.zeros((0, height, width), dtype=torch.uint8) + return masks + + +class ConvertCocoPolysToMask(object): + def __init__(self, return_masks=False): + self.return_masks = return_masks + + def __call__(self, image, target): + w, h = image.size + + image_id = target["image_id"] + image_id = torch.tensor([image_id]) + + anno = target["annotations"] + + anno = [obj for obj in anno if "iscrowd" not in obj or obj["iscrowd"] == 0] + + boxes = [obj["bbox"] for obj in anno] + # guard against no boxes via resizing + boxes = torch.as_tensor(boxes, dtype=torch.float32).reshape(-1, 4) + boxes[:, 2:] += boxes[:, :2] + boxes[:, 0::2].clamp_(min=0, max=w) + boxes[:, 1::2].clamp_(min=0, max=h) + + classes = [obj["category_id"] for obj in anno] + classes = torch.tensor(classes, dtype=torch.int64) + + if self.return_masks: + segmentations = [obj["segmentation"] for obj in anno] + masks = convert_coco_poly_to_mask(segmentations, h, w) + + keypoints = None + if anno and "keypoints" in anno[0]: + keypoints = [obj["keypoints"] for obj in anno] + keypoints = torch.as_tensor(keypoints, dtype=torch.float32) + num_keypoints = keypoints.shape[0] + if num_keypoints: + keypoints = keypoints.view(num_keypoints, -1, 3) + + keep = (boxes[:, 3] > boxes[:, 1]) & (boxes[:, 2] > boxes[:, 0]) + boxes = boxes[keep] + classes = classes[keep] + if self.return_masks: + masks = masks[keep] + if keypoints is not None: + keypoints = keypoints[keep] + + target = {} + target["boxes"] = boxes + target["labels"] = classes + if self.return_masks: + target["masks"] = masks + target["image_id"] = image_id + if keypoints is not None: + target["keypoints"] = keypoints + + # for conversion to coco api + area = torch.tensor([obj["area"] for obj in anno]) + iscrowd = torch.tensor( + [obj["iscrowd"] if "iscrowd" in obj else 0 for obj in anno] + ) + target["area"] = area[keep] + target["iscrowd"] = iscrowd[keep] + + target["orig_size"] = torch.as_tensor([int(h), int(w)]) + target["size"] = torch.as_tensor([int(h), int(w)]) + + return image, target + + +def make_coco_transforms(image_set, fix_size=False, strong_aug=False, args=None): + normalize = T.Compose( + [T.ToTensor(), T.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])] + ) + + # config the params for data aug + scales = [480, 512, 544, 576, 608, 640, 672, 704, 736, 768, 800] + max_size = 1333 + scales2_resize = [400, 500, 600] + scales2_crop = [384, 600] + + # update args from config files + scales = getattr(args, "data_aug_scales", scales) + max_size = getattr(args, "data_aug_max_size", max_size) + scales2_resize = getattr(args, "data_aug_scales2_resize", scales2_resize) + scales2_crop = getattr(args, "data_aug_scales2_crop", scales2_crop) + + # resize them + data_aug_scale_overlap = getattr(args, "data_aug_scale_overlap", None) + if data_aug_scale_overlap is not None and data_aug_scale_overlap > 0: + data_aug_scale_overlap = float(data_aug_scale_overlap) + scales = [int(i * data_aug_scale_overlap) for i in scales] + max_size = int(max_size * data_aug_scale_overlap) + scales2_resize = [int(i * data_aug_scale_overlap) for i in scales2_resize] + scales2_crop = [int(i * data_aug_scale_overlap) for i in scales2_crop] + + datadict_for_print = { + "scales": scales, + "max_size": max_size, + "scales2_resize": scales2_resize, + "scales2_crop": scales2_crop, + } + # print("data_aug_params:", json.dumps(datadict_for_print, indent=2)) + + if image_set == "train": + if fix_size: + return T.Compose( + [ + T.RandomHorizontalFlip(), + T.RandomResize([(max_size, max(scales))]), + # T.RandomResize([(512, 512)]), + normalize, + ] + ) + + if strong_aug: + import datasets.sltransform as SLT + + return T.Compose( + [ + T.RandomHorizontalFlip(), + T.RandomSelect( + T.RandomResize(scales, max_size=max_size), + T.Compose( + [ + T.RandomResize(scales2_resize), + T.RandomSizeCrop(*scales2_crop), + T.RandomResize(scales, max_size=max_size), + ] + ), + ), + SLT.RandomSelectMulti( + [ + SLT.RandomCrop(), + SLT.LightingNoise(), + SLT.AdjustBrightness(2), + SLT.AdjustContrast(2), + ] + ), + normalize, + ] + ) + + return T.Compose( + [ + T.RandomHorizontalFlip(), + T.RandomSelect( + T.RandomResize(scales, max_size=max_size), + T.Compose( + [ + T.RandomResize(scales2_resize), + T.RandomSizeCrop(*scales2_crop), + T.RandomResize(scales, max_size=max_size), + ] + ), + ), + normalize, + ] + ) + + if image_set in ["val", "eval_debug", "train_reg", "test"]: + if os.environ.get("GFLOPS_DEBUG_SHILONG", False) == "INFO": + print("Under debug mode for flops calculation only!!!!!!!!!!!!!!!!") + return T.Compose( + [ + T.ResizeDebug((1280, 800)), + normalize, + ] + ) + + print("max(scales): " + str(max(scales))) + + return T.Compose( + [ + T.RandomResize([max(scales)], max_size=max_size), + normalize, + ] + ) + + raise ValueError(f"unknown {image_set}") + + +def get_aux_target_hacks_list(image_set, args): + if args.modelname in ["q2bs_mask", "q2bs"]: + aux_target_hacks_list = [ + label2compat(), + label_compat2onehot(), + RandomSelectBoxes(num_class=args.num_classes), + ] + if args.masked_data and image_set == "train": + # aux_target_hacks_list.append() + aux_target_hacks_list.append(MaskCrop()) + elif args.modelname in [ + "q2bm_v2", + "q2bs_ce", + "q2op", + "q2ofocal", + "q2opclip", + "q2ocqonly", + ]: + aux_target_hacks_list = [ + label2compat(), + label_compat2onehot(), + box_label_catter(), + RandomSelectBoxlabels( + num_classes=args.num_classes, + prob_first_item=args.prob_first_item, + prob_random_item=args.prob_random_item, + prob_last_item=args.prob_last_item, + prob_stop_sign=args.prob_stop_sign, + ), + BboxPertuber(max_ratio=0.02, generate_samples=1000), + ] + elif args.modelname in ["q2omask", "q2osa"]: + if args.coco_aug: + aux_target_hacks_list = [ + label2compat(), + label_compat2onehot(), + box_label_catter(), + RandomSelectBoxlabels( + num_classes=args.num_classes, + prob_first_item=args.prob_first_item, + prob_random_item=args.prob_random_item, + prob_last_item=args.prob_last_item, + prob_stop_sign=args.prob_stop_sign, + ), + RandomDrop(p=0.2), + BboxPertuber(max_ratio=0.02, generate_samples=1000), + RandomCutout(factor=0.5), + ] + else: + aux_target_hacks_list = [ + label2compat(), + label_compat2onehot(), + box_label_catter(), + RandomSelectBoxlabels( + num_classes=args.num_classes, + prob_first_item=args.prob_first_item, + prob_random_item=args.prob_random_item, + prob_last_item=args.prob_last_item, + prob_stop_sign=args.prob_stop_sign, + ), + BboxPertuber(max_ratio=0.02, generate_samples=1000), + ] + else: + aux_target_hacks_list = None + + return aux_target_hacks_list + + +def build(image_set, args, datasetinfo): + img_folder = datasetinfo["root"] + ann_file = datasetinfo["anno"] + + # copy to local path + if os.environ.get("DATA_COPY_SHILONG") == "INFO": + preparing_dataset( + dict(img_folder=img_folder, ann_file=ann_file), image_set, args + ) + + try: + strong_aug = args.strong_aug + except: + strong_aug = False + print(img_folder, ann_file) + dataset = CocoDetection( + img_folder, + ann_file, + transforms=make_coco_transforms( + image_set, fix_size=args.fix_size, strong_aug=strong_aug, args=args + ), + return_masks=args.masks, + aux_target_hacks=None, + ) + return dataset + + +if __name__ == "__main__": + # Objects365 Val example + dataset_o365 = CocoDetection( + "/path/Objects365/train/", + "/path/Objects365/slannos/anno_preprocess_train_v2.json", + transforms=None, + return_masks=False, + ) + print("len(dataset_o365):", len(dataset_o365)) diff --git a/datasets/coco_eval.py b/datasets/coco_eval.py new file mode 100644 index 0000000000000000000000000000000000000000..74bbc0b32b5e4dbe40cddae7ff9add26e3f910f9 --- /dev/null +++ b/datasets/coco_eval.py @@ -0,0 +1,266 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved +""" +COCO evaluator that works in distributed mode. + +Mostly copy-paste from https://github.com/pytorch/vision/blob/edfd5a7/references/detection/coco_eval.py +The difference is that there is less copy-pasting from pycocotools +in the end of the file, as python3 can suppress prints with contextlib +""" +import os +import contextlib +import copy +import numpy as np +import torch + +from pycocotools.cocoeval import COCOeval +from pycocotools.coco import COCO +import pycocotools.mask as mask_util + +from util.misc import all_gather + + +class CocoEvaluator(object): + def __init__(self, coco_gt, iou_types, useCats=True): + assert isinstance(iou_types, (list, tuple)) + coco_gt = copy.deepcopy(coco_gt) + self.coco_gt = coco_gt + + self.iou_types = iou_types + self.coco_eval = {} + for iou_type in iou_types: + self.coco_eval[iou_type] = COCOeval(coco_gt, iouType=iou_type) + self.coco_eval[iou_type].useCats = useCats + + self.img_ids = [] + self.eval_imgs = {k: [] for k in iou_types} + self.useCats = useCats + + def update(self, predictions): + img_ids = list(np.unique(list(predictions.keys()))) + self.img_ids.extend(img_ids) + + for iou_type in self.iou_types: + results = self.prepare(predictions, iou_type) + + # suppress pycocotools prints + with open(os.devnull, 'w') as devnull: + with contextlib.redirect_stdout(devnull): + coco_dt = COCO.loadRes(self.coco_gt, results) if results else COCO() + coco_eval = self.coco_eval[iou_type] + + coco_eval.cocoDt = coco_dt + coco_eval.params.imgIds = list(img_ids) + coco_eval.params.useCats = self.useCats + img_ids, eval_imgs = evaluate(coco_eval) + + self.eval_imgs[iou_type].append(eval_imgs) + + def synchronize_between_processes(self): + for iou_type in self.iou_types: + self.eval_imgs[iou_type] = np.concatenate(self.eval_imgs[iou_type], 2) + create_common_coco_eval(self.coco_eval[iou_type], self.img_ids, self.eval_imgs[iou_type]) + + def accumulate(self): + for coco_eval in self.coco_eval.values(): + coco_eval.accumulate() + + def summarize(self): + for iou_type, coco_eval in self.coco_eval.items(): + print("IoU metric: {}".format(iou_type)) + coco_eval.summarize() + + def prepare(self, predictions, iou_type): + if iou_type == "bbox": + return self.prepare_for_coco_detection(predictions) + elif iou_type == "segm": + return self.prepare_for_coco_segmentation(predictions) + elif iou_type == "keypoints": + return self.prepare_for_coco_keypoint(predictions) + else: + raise ValueError("Unknown iou type {}".format(iou_type)) + + def prepare_for_coco_detection(self, predictions): + coco_results = [] + for original_id, prediction in predictions.items(): + if len(prediction) == 0: + continue + + boxes = prediction["boxes"] + boxes = convert_to_xywh(boxes).tolist() + if not isinstance(prediction["scores"], list): + scores = prediction["scores"].tolist() + else: + scores = prediction["scores"] + if not isinstance(prediction["labels"], list): + labels = prediction["labels"].tolist() + else: + labels = prediction["labels"] + + + try: + coco_results.extend( + [ + { + "image_id": original_id, + "category_id": labels[k], + "bbox": box, + "score": scores[k], + } + for k, box in enumerate(boxes) + ] + ) + except: + import ipdb; ipdb.set_trace() + return coco_results + + def prepare_for_coco_segmentation(self, predictions): + coco_results = [] + for original_id, prediction in predictions.items(): + if len(prediction) == 0: + continue + + scores = prediction["scores"] + labels = prediction["labels"] + masks = prediction["masks"] + + masks = masks > 0.5 + + scores = prediction["scores"].tolist() + labels = prediction["labels"].tolist() + + rles = [ + mask_util.encode(np.array(mask[0, :, :, np.newaxis], dtype=np.uint8, order="F"))[0] + for mask in masks + ] + for rle in rles: + rle["counts"] = rle["counts"].decode("utf-8") + + coco_results.extend( + [ + { + "image_id": original_id, + "category_id": labels[k], + "segmentation": rle, + "score": scores[k], + } + for k, rle in enumerate(rles) + ] + ) + return coco_results + + def prepare_for_coco_keypoint(self, predictions): + coco_results = [] + for original_id, prediction in predictions.items(): + if len(prediction) == 0: + continue + + boxes = prediction["boxes"] + boxes = convert_to_xywh(boxes).tolist() + scores = prediction["scores"].tolist() + labels = prediction["labels"].tolist() + keypoints = prediction["keypoints"] + keypoints = keypoints.flatten(start_dim=1).tolist() + + coco_results.extend( + [ + { + "image_id": original_id, + "category_id": labels[k], + 'keypoints': keypoint, + "score": scores[k], + } + for k, keypoint in enumerate(keypoints) + ] + ) + return coco_results + + +def convert_to_xywh(boxes): + xmin, ymin, xmax, ymax = boxes.unbind(1) + return torch.stack((xmin, ymin, xmax - xmin, ymax - ymin), dim=1) + + +def merge(img_ids, eval_imgs): + all_img_ids = all_gather(img_ids) + all_eval_imgs = all_gather(eval_imgs) + + merged_img_ids = [] + for p in all_img_ids: + merged_img_ids.extend(p) + + merged_eval_imgs = [] + for p in all_eval_imgs: + merged_eval_imgs.append(p) + + merged_img_ids = np.array(merged_img_ids) + merged_eval_imgs = np.concatenate(merged_eval_imgs, 2) + + # keep only unique (and in sorted order) images + merged_img_ids, idx = np.unique(merged_img_ids, return_index=True) + merged_eval_imgs = merged_eval_imgs[..., idx] + + return merged_img_ids, merged_eval_imgs + + +def create_common_coco_eval(coco_eval, img_ids, eval_imgs): + img_ids, eval_imgs = merge(img_ids, eval_imgs) + img_ids = list(img_ids) + eval_imgs = list(eval_imgs.flatten()) + + coco_eval.evalImgs = eval_imgs + coco_eval.params.imgIds = img_ids + coco_eval._paramsEval = copy.deepcopy(coco_eval.params) + + +################################################################# +# From pycocotools, just removed the prints and fixed +# a Python3 bug about unicode not defined +################################################################# + + +def evaluate(self): + ''' + Run per image evaluation on given images and store results (a list of dict) in self.evalImgs + :return: None + ''' + p = self.params + # add backward compatibility if useSegm is specified in params + if p.useSegm is not None: + p.iouType = 'segm' if p.useSegm == 1 else 'bbox' + print('useSegm (deprecated) is not None. Running {} evaluation'.format(p.iouType)) + p.imgIds = list(np.unique(p.imgIds)) + if p.useCats: + p.catIds = list(np.unique(p.catIds)) + p.maxDets = sorted(p.maxDets) + self.params = p + + self._prepare() + # loop through images, area range, max detection number + catIds = p.catIds if p.useCats else [-1] + + if p.iouType == 'segm' or p.iouType == 'bbox': + computeIoU = self.computeIoU + elif p.iouType == 'keypoints': + computeIoU = self.computeOks + self.ious = { + (imgId, catId): computeIoU(imgId, catId) + for imgId in p.imgIds + for catId in catIds} + + evaluateImg = self.evaluateImg + maxDet = p.maxDets[-1] + evalImgs = [ + evaluateImg(imgId, catId, areaRng, maxDet) + for catId in catIds + for areaRng in p.areaRng + for imgId in p.imgIds + ] + # this is NOT in the pycocotools code, but could be done outside + evalImgs = np.asarray(evalImgs).reshape(len(catIds), len(p.areaRng), len(p.imgIds)) + self._paramsEval = copy.deepcopy(self.params) + + return p.imgIds, evalImgs + +################################################################# +# end of straight copy from pycocotools, just removing the prints +################################################################# diff --git a/datasets/coco_panoptic.py b/datasets/coco_panoptic.py new file mode 100644 index 0000000000000000000000000000000000000000..b24f615c2faa14b422829e2edad996e2b5b84248 --- /dev/null +++ b/datasets/coco_panoptic.py @@ -0,0 +1,99 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved +import json +from pathlib import Path + +import numpy as np +import torch +from PIL import Image + +from panopticapi.utils import rgb2id +from util.box_ops import masks_to_boxes + +from .coco import make_coco_transforms + + +class CocoPanoptic: + def __init__(self, img_folder, ann_folder, ann_file, transforms=None, return_masks=True): + with open(ann_file, 'r') as f: + self.coco = json.load(f) + + # sort 'images' field so that they are aligned with 'annotations' + # i.e., in alphabetical order + self.coco['images'] = sorted(self.coco['images'], key=lambda x: x['id']) + # sanity check + if "annotations" in self.coco: + for img, ann in zip(self.coco['images'], self.coco['annotations']): + assert img['file_name'][:-4] == ann['file_name'][:-4] + + self.img_folder = img_folder + self.ann_folder = ann_folder + self.ann_file = ann_file + self.transforms = transforms + self.return_masks = return_masks + + def __getitem__(self, idx): + ann_info = self.coco['annotations'][idx] if "annotations" in self.coco else self.coco['images'][idx] + img_path = Path(self.img_folder) / ann_info['file_name'].replace('.png', '.jpg') + ann_path = Path(self.ann_folder) / ann_info['file_name'] + + img = Image.open(img_path).convert('RGB') + w, h = img.size + if "segments_info" in ann_info: + masks = np.asarray(Image.open(ann_path), dtype=np.uint32) + masks = rgb2id(masks) + + ids = np.array([ann['id'] for ann in ann_info['segments_info']]) + masks = masks == ids[:, None, None] + + masks = torch.as_tensor(masks, dtype=torch.uint8) + labels = torch.tensor([ann['category_id'] for ann in ann_info['segments_info']], dtype=torch.int64) + + target = {} + target['image_id'] = torch.tensor([ann_info['image_id'] if "image_id" in ann_info else ann_info["id"]]) + if self.return_masks: + target['masks'] = masks + target['labels'] = labels + + target["boxes"] = masks_to_boxes(masks) + + target['size'] = torch.as_tensor([int(h), int(w)]) + target['orig_size'] = torch.as_tensor([int(h), int(w)]) + if "segments_info" in ann_info: + for name in ['iscrowd', 'area']: + target[name] = torch.tensor([ann[name] for ann in ann_info['segments_info']]) + + if self.transforms is not None: + img, target = self.transforms(img, target) + + return img, target + + def __len__(self): + return len(self.coco['images']) + + def get_height_and_width(self, idx): + img_info = self.coco['images'][idx] + height = img_info['height'] + width = img_info['width'] + return height, width + + +def build(image_set, args): + img_folder_root = Path(args.coco_path) + ann_folder_root = Path(args.coco_panoptic_path) + assert img_folder_root.exists(), f'provided COCO path {img_folder_root} does not exist' + assert ann_folder_root.exists(), f'provided COCO path {ann_folder_root} does not exist' + mode = 'panoptic' + PATHS = { + "train": ("train2017", Path("annotations") / f'{mode}_train2017.json'), + "val": ("val2017", Path("annotations") / f'{mode}_val2017.json'), + } + + img_folder, ann_file = PATHS[image_set] + img_folder_path = img_folder_root / img_folder + ann_folder = ann_folder_root / f'{mode}_{img_folder}' + ann_file = ann_folder_root / ann_file + + dataset = CocoPanoptic(img_folder_path, ann_folder, ann_file, + transforms=make_coco_transforms(image_set), return_masks=args.masks) + + return dataset diff --git a/datasets/cocogrounding_eval.py b/datasets/cocogrounding_eval.py new file mode 100644 index 0000000000000000000000000000000000000000..32f86a50a8f1daf36135edefd20574e022fe5ff6 --- /dev/null +++ b/datasets/cocogrounding_eval.py @@ -0,0 +1,271 @@ +# ------------------------------------------------------------------------ +# Grounding DINO. Midified by Shilong Liu. +# url: https://github.com/IDEA-Research/GroundingDINO +# Copyright (c) 2023 IDEA. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------ +# Copyright (c) Aishwarya Kamath & Nicolas Carion. Licensed under the Apache License 2.0. All Rights Reserved +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved +""" +COCO evaluator that works in distributed mode. + +Mostly copy-paste from https://github.com/pytorch/vision/blob/edfd5a7/references/detection/coco_eval.py +The difference is that there is less copy-pasting from pycocotools +in the end of the file, as python3 can suppress prints with contextlib +""" +import contextlib +import copy +import os + +import numpy as np +import pycocotools.mask as mask_util +import torch +from pycocotools.coco import COCO +from pycocotools.cocoeval import COCOeval + +from groundingdino.util.misc import all_gather + + +class CocoGroundingEvaluator(object): + def __init__(self, coco_gt, iou_types, useCats=True): + assert isinstance(iou_types, (list, tuple)) + coco_gt = copy.deepcopy(coco_gt) + self.coco_gt = coco_gt + + self.iou_types = iou_types + self.coco_eval = {} + for iou_type in iou_types: + self.coco_eval[iou_type] = COCOeval(coco_gt, iouType=iou_type) + self.coco_eval[iou_type].useCats = useCats + + self.img_ids = [] + self.eval_imgs = {k: [] for k in iou_types} + self.useCats = useCats + + def update(self, predictions): + img_ids = list(np.unique(list(predictions.keys()))) + self.img_ids.extend(img_ids) + # import pdb;pdb.set_trace() + for iou_type in self.iou_types: + results = self.prepare(predictions, iou_type) + + # suppress pycocotools prints + with open(os.devnull, "w") as devnull: + with contextlib.redirect_stdout(devnull): + coco_dt = COCO.loadRes(self.coco_gt, results) if results else COCO() + + coco_eval = self.coco_eval[iou_type] + + coco_eval.cocoDt = coco_dt + coco_eval.params.imgIds = list(img_ids) + coco_eval.params.useCats = self.useCats + img_ids, eval_imgs = evaluate(coco_eval) + + self.eval_imgs[iou_type].append(eval_imgs) + + def synchronize_between_processes(self): + for iou_type in self.iou_types: + self.eval_imgs[iou_type] = np.concatenate(self.eval_imgs[iou_type], 2) + create_common_coco_eval(self.coco_eval[iou_type], self.img_ids, self.eval_imgs[iou_type]) + + def accumulate(self): + for coco_eval in self.coco_eval.values(): + coco_eval.accumulate() + + def summarize(self): + for iou_type, coco_eval in self.coco_eval.items(): + print("IoU metric: {}".format(iou_type)) + coco_eval.summarize() + + def prepare(self, predictions, iou_type): + if iou_type == "bbox": + return self.prepare_for_coco_detection(predictions) + elif iou_type == "segm": + return self.prepare_for_coco_segmentation(predictions) + elif iou_type == "keypoints": + return self.prepare_for_coco_keypoint(predictions) + else: + raise ValueError("Unknown iou type {}".format(iou_type)) + + def prepare_for_coco_detection(self, predictions): + coco_results = [] + for original_id, prediction in predictions.items(): + if len(prediction) == 0: + continue + + boxes = prediction["boxes"] + boxes = convert_to_xywh(boxes).tolist() + scores = prediction["scores"].tolist() + labels = prediction["labels"].tolist() + + coco_results.extend( + [ + { + "image_id": original_id, + "category_id": labels[k], + "bbox": box, + "score": scores[k], + } + for k, box in enumerate(boxes) + ] + ) + return coco_results + + def prepare_for_coco_segmentation(self, predictions): + coco_results = [] + for original_id, prediction in predictions.items(): + if len(prediction) == 0: + continue + + scores = prediction["scores"] + labels = prediction["labels"] + masks = prediction["masks"] + + masks = masks > 0.5 + + scores = prediction["scores"].tolist() + labels = prediction["labels"].tolist() + + rles = [ + mask_util.encode(np.array(mask[0, :, :, np.newaxis], dtype=np.uint8, order="F"))[0] + for mask in masks + ] + for rle in rles: + rle["counts"] = rle["counts"].decode("utf-8") + + coco_results.extend( + [ + { + "image_id": original_id, + "category_id": labels[k], + "segmentation": rle, + "score": scores[k], + } + for k, rle in enumerate(rles) + ] + ) + return coco_results + + def prepare_for_coco_keypoint(self, predictions): + coco_results = [] + for original_id, prediction in predictions.items(): + if len(prediction) == 0: + continue + + boxes = prediction["boxes"] + boxes = convert_to_xywh(boxes).tolist() + scores = prediction["scores"].tolist() + labels = prediction["labels"].tolist() + keypoints = prediction["keypoints"] + keypoints = keypoints.flatten(start_dim=1).tolist() + + coco_results.extend( + [ + { + "image_id": original_id, + "category_id": labels[k], + "keypoints": keypoint, + "score": scores[k], + } + for k, keypoint in enumerate(keypoints) + ] + ) + return coco_results + + +def convert_to_xywh(boxes): + xmin, ymin, xmax, ymax = boxes.unbind(1) + return torch.stack((xmin, ymin, xmax - xmin, ymax - ymin), dim=1) + + +def merge(img_ids, eval_imgs): + all_img_ids = all_gather(img_ids) + all_eval_imgs = all_gather(eval_imgs) + + merged_img_ids = [] + for p in all_img_ids: + merged_img_ids.extend(p) + + merged_eval_imgs = [] + for p in all_eval_imgs: + merged_eval_imgs.append(p) + + merged_img_ids = np.array(merged_img_ids) + merged_eval_imgs = np.concatenate(merged_eval_imgs, 2) + + # keep only unique (and in sorted order) images + merged_img_ids, idx = np.unique(merged_img_ids, return_index=True) + merged_eval_imgs = merged_eval_imgs[..., idx] + + return merged_img_ids, merged_eval_imgs + + +def create_common_coco_eval(coco_eval, img_ids, eval_imgs): + img_ids, eval_imgs = merge(img_ids, eval_imgs) + img_ids = list(img_ids) + eval_imgs = list(eval_imgs.flatten()) + + coco_eval.evalImgs = eval_imgs + coco_eval.params.imgIds = img_ids + coco_eval._paramsEval = copy.deepcopy(coco_eval.params) + + +################################################################# +# From pycocotools, just removed the prints and fixed +# a Python3 bug about unicode not defined +################################################################# + + +def evaluate(self): + """ + Run per image evaluation on given images and store results (a list of dict) in self.evalImgs + :return: None + """ + # tic = time.time() + # print('Running per image evaluation...') + + # import pdb;pdb.set_trace() + p = self.params + # add backward compatibility if useSegm is specified in params + if p.useSegm is not None: + p.iouType = "segm" if p.useSegm == 1 else "bbox" + print("useSegm (deprecated) is not None. Running {} evaluation".format(p.iouType)) + # print('Evaluate annotation type *{}*'.format(p.iouType)) + p.imgIds = list(np.unique(p.imgIds)) + if p.useCats: + p.catIds = list(np.unique(p.catIds)) + p.maxDets = sorted(p.maxDets) + self.params = p + + self._prepare() + # loop through images, area range, max detection number + catIds = p.catIds if p.useCats else [-1] + + if p.iouType == "segm" or p.iouType == "bbox": + computeIoU = self.computeIoU + elif p.iouType == "keypoints": + computeIoU = self.computeOks + self.ious = { + (imgId, catId): computeIoU(imgId, catId) + for imgId in p.imgIds + for catId in catIds} + + evaluateImg = self.evaluateImg + maxDet = p.maxDets[-1] + evalImgs = [ + evaluateImg(imgId, catId, areaRng, maxDet) + for catId in catIds + for areaRng in p.areaRng + for imgId in p.imgIds + ] + # this is NOT in the pycocotools code, but could be done outside + evalImgs = np.asarray(evalImgs).reshape(len(catIds), len(p.areaRng), len(p.imgIds)) + self._paramsEval = copy.deepcopy(self.params) + # toc = time.time() + # print('DONE (t={:0.2f}s).'.format(toc-tic)) + return p.imgIds, evalImgs + + +################################################################# +# end of straight copy from pycocotools, just removing the prints +################################################################# diff --git a/datasets/data_util.py b/datasets/data_util.py new file mode 100644 index 0000000000000000000000000000000000000000..8019ccd7348f87640620c52b7a3f8a1c266859a7 --- /dev/null +++ b/datasets/data_util.py @@ -0,0 +1,170 @@ +import os +import os.path as osp +import shutil +import time +import datetime + +import torch + +from util.slconfig import SLConfig + +class Error(OSError): + pass + +def slcopytree(src, dst, symlinks=False, ignore=None, copy_function=shutil.copyfile, + ignore_dangling_symlinks=False): + """ + modified from shutil.copytree without copystat. + + Recursively copy a directory tree. + + The destination directory must not already exist. + If exception(s) occur, an Error is raised with a list of reasons. + + If the optional symlinks flag is true, symbolic links in the + source tree result in symbolic links in the destination tree; if + it is false, the contents of the files pointed to by symbolic + links are copied. If the file pointed by the symlink doesn't + exist, an exception will be added in the list of errors raised in + an Error exception at the end of the copy process. + + You can set the optional ignore_dangling_symlinks flag to true if you + want to silence this exception. Notice that this has no effect on + platforms that don't support os.symlink. + + The optional ignore argument is a callable. If given, it + is called with the `src` parameter, which is the directory + being visited by copytree(), and `names` which is the list of + `src` contents, as returned by os.listdir(): + + callable(src, names) -> ignored_names + + Since copytree() is called recursively, the callable will be + called once for each directory that is copied. It returns a + list of names relative to the `src` directory that should + not be copied. + + The optional copy_function argument is a callable that will be used + to copy each file. It will be called with the source path and the + destination path as arguments. By default, copy2() is used, but any + function that supports the same signature (like copy()) can be used. + + """ + errors = [] + if os.path.isdir(src): + names = os.listdir(src) + if ignore is not None: + ignored_names = ignore(src, names) + else: + ignored_names = set() + + os.makedirs(dst) + for name in names: + if name in ignored_names: + continue + srcname = os.path.join(src, name) + dstname = os.path.join(dst, name) + try: + if os.path.islink(srcname): + linkto = os.readlink(srcname) + if symlinks: + # We can't just leave it to `copy_function` because legacy + # code with a custom `copy_function` may rely on copytree + # doing the right thing. + os.symlink(linkto, dstname) + else: + # ignore dangling symlink if the flag is on + if not os.path.exists(linkto) and ignore_dangling_symlinks: + continue + # otherwise let the copy occurs. copy2 will raise an error + if os.path.isdir(srcname): + slcopytree(srcname, dstname, symlinks, ignore, + copy_function) + else: + copy_function(srcname, dstname) + elif os.path.isdir(srcname): + slcopytree(srcname, dstname, symlinks, ignore, copy_function) + else: + # Will raise a SpecialFileError for unsupported file types + copy_function(srcname, dstname) + # catch the Error from the recursive copytree so that we can + # continue with other files + except Error as err: + errors.extend(err.args[0]) + except OSError as why: + errors.append((srcname, dstname, str(why))) + else: + copy_function(src, dst) + + if errors: + raise Error(errors) + return dst + +def check_and_copy(src_path, tgt_path): + if os.path.exists(tgt_path): + return None + + return slcopytree(src_path, tgt_path) + + +def remove(srcpath): + if os.path.isdir(srcpath): + return shutil.rmtree(srcpath) + else: + return os.remove(srcpath) + + +def preparing_dataset(pathdict, image_set, args): + start_time = time.time() + dataset_file = args.dataset_file + data_static_info = SLConfig.fromfile('util/static_data_path.py') + static_dict = data_static_info[dataset_file][image_set] + + copyfilelist = [] + for k,tgt_v in pathdict.items(): + if os.path.exists(tgt_v): + if args.local_rank == 0: + print("path <{}> exist. remove it!".format(tgt_v)) + remove(tgt_v) + # continue + + if args.local_rank == 0: + src_v = static_dict[k] + assert isinstance(src_v, str) + if src_v.endswith('.zip'): + # copy + cp_tgt_dir = os.path.dirname(tgt_v) + filename = os.path.basename(src_v) + cp_tgt_path = os.path.join(cp_tgt_dir, filename) + print('Copy from <{}> to <{}>.'.format(src_v, cp_tgt_path)) + os.makedirs(cp_tgt_dir, exist_ok=True) + check_and_copy(src_v, cp_tgt_path) + + # unzip + import zipfile + print("Starting unzip <{}>".format(cp_tgt_path)) + with zipfile.ZipFile(cp_tgt_path, 'r') as zip_ref: + zip_ref.extractall(os.path.dirname(cp_tgt_path)) + + copyfilelist.append(cp_tgt_path) + copyfilelist.append(tgt_v) + else: + print('Copy from <{}> to <{}>.'.format(src_v, tgt_v)) + os.makedirs(os.path.dirname(tgt_v), exist_ok=True) + check_and_copy(src_v, tgt_v) + copyfilelist.append(tgt_v) + + if len(copyfilelist) == 0: + copyfilelist = None + args.copyfilelist = copyfilelist + + if args.distributed: + torch.distributed.barrier() + total_time = time.time() - start_time + if copyfilelist: + total_time_str = str(datetime.timedelta(seconds=int(total_time))) + print('Data copy time {}'.format(total_time_str)) + return copyfilelist + + + \ No newline at end of file diff --git a/datasets/dataset.py b/datasets/dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..5655c00ccdbf0b09d51da8dfa3593a2bf9aee527 --- /dev/null +++ b/datasets/dataset.py @@ -0,0 +1,44 @@ +from __future__ import print_function + +import torch +import torchvision.datasets as datasets +from torch.utils.data import Dataset +from PIL import Image +from .tsv_io import TSVFile +import numpy as np +import base64 +import io + + +class TSVDataset(Dataset): + """ TSV dataset for ImageNet 1K training + """ + def __init__(self, tsv_file, transform=None, target_transform=None): + self.tsv = TSVFile(tsv_file) + self.transform = transform + self.target_transform = target_transform + + def __getitem__(self, index): + """ + Args: + index (int): Index + Returns: + tuple: (image, target) where target is class_index of the target class. + """ + row = self.tsv.seek(index) + image_data = base64.b64decode(row[-1]) + image = Image.open(io.BytesIO(image_data)) + image = image.convert('RGB') + target = int(row[1]) + + if self.transform is not None: + img = self.transform(image) + else: + img = image + if self.target_transform is not None: + target = self.target_transform(target) + + return img, target + + def __len__(self): + return self.tsv.num_rows() diff --git a/datasets/odvg.py b/datasets/odvg.py new file mode 100644 index 0000000000000000000000000000000000000000..f30cb8f76c3034fe9df50bc8aef8c6489d65e056 --- /dev/null +++ b/datasets/odvg.py @@ -0,0 +1,288 @@ +from torchvision.datasets.vision import VisionDataset +import os.path +from typing import Callable, Optional +import json +from PIL import Image +import torch +import random +import os, sys + +sys.path.append(os.path.dirname(sys.path[0])) + +import datasets.transforms as T + + +class ODVGDataset(VisionDataset): + """ + Args: + root (string): Root directory where images are downloaded to. + anno (string): Path to json annotation file. + label_map_anno (string): Path to json label mapping file. Only for Object Detection + transform (callable, optional): A function/transform that takes in an PIL image + and returns a transformed version. E.g, ``transforms.PILToTensor`` + target_transform (callable, optional): A function/transform that takes in the + target and transforms it. + transforms (callable, optional): A function/transform that takes input sample and its target as entry + and returns a transformed version. + """ + + def __init__( + self, + root: str, + anno: str, + label_map_anno: str = None, + max_labels: int = 80, + transform: Optional[Callable] = None, + target_transform: Optional[Callable] = None, + transforms: Optional[Callable] = None, + ) -> None: + super().__init__(root, transforms, transform, target_transform) + self.root = root + self.dataset_mode = "OD" if label_map_anno else "VG" + self.max_labels = max_labels + if self.dataset_mode == "OD": + self.load_label_map(label_map_anno) + self._load_metas(anno) + self.get_dataset_info() + + def load_label_map(self, label_map_anno): + with open(label_map_anno, "r") as file: + self.label_map = json.load(file) + self.label_index = set(self.label_map.keys()) + + def _load_metas(self, anno): + with open(anno, "r") as f: + self.metas = [json.loads(line) for line in f] + + def get_dataset_info(self): + print(f" == total images: {len(self)}") + if self.dataset_mode == "OD": + print(f" == total labels: {len(self.label_map)}") + + def __getitem__(self, index: int): + meta = self.metas[index] + rel_path = meta["filename"] + abs_path = os.path.join(self.root, rel_path) + if not os.path.exists(abs_path): + raise FileNotFoundError(f"{abs_path} not found.") + image = Image.open(abs_path).convert("RGB") + exemplars = torch.tensor(meta["exemplars"], dtype=torch.int64) + w, h = image.size + if self.dataset_mode == "OD": + anno = meta["detection"] + instances = [obj for obj in anno["instances"]] + boxes = [obj["bbox"] for obj in instances] + # generate vg_labels + # pos bbox labels + ori_classes = [str(obj["label"]) for obj in instances] + pos_labels = set(ori_classes) + # neg bbox labels + neg_labels = self.label_index.difference(pos_labels) + + vg_labels = list(pos_labels) + num_to_add = min(len(neg_labels), self.max_labels - len(pos_labels)) + if num_to_add > 0: + vg_labels.extend(random.sample(neg_labels, num_to_add)) + + # shuffle + for i in range(len(vg_labels) - 1, 0, -1): + j = random.randint(0, i) + vg_labels[i], vg_labels[j] = vg_labels[j], vg_labels[i] + + caption_list = [self.label_map[lb] for lb in vg_labels] + caption_dict = {item: index for index, item in enumerate(caption_list)} + + caption = " . ".join(caption_list) + " ." + classes = [ + caption_dict[self.label_map[str(obj["label"])]] for obj in instances + ] + boxes = torch.as_tensor(boxes, dtype=torch.float32).reshape(-1, 4) + classes = torch.tensor(classes, dtype=torch.int64) + elif self.dataset_mode == "VG": + anno = meta["grounding"] + instances = [obj for obj in anno["regions"]] + boxes = [obj["bbox"] for obj in instances] + caption_list = [obj["phrase"] for obj in instances] + c = list(zip(boxes, caption_list)) + random.shuffle(c) + boxes[:], caption_list[:] = zip(*c) + uni_caption_list = list(set(caption_list)) + label_map = {} + for idx in range(len(uni_caption_list)): + label_map[uni_caption_list[idx]] = idx + classes = [label_map[cap] for cap in caption_list] + caption = " . ".join(uni_caption_list) + " ." + boxes = torch.as_tensor(boxes, dtype=torch.float32).reshape(-1, 4) + classes = torch.tensor(classes, dtype=torch.int64) + caption_list = uni_caption_list + target = {} + target["size"] = torch.as_tensor([int(h), int(w)]) + target["cap_list"] = caption_list + target["caption"] = caption + target["boxes"] = boxes + target["labels"] = classes + target["exemplars"] = exemplars + target["labels_uncropped"] = torch.clone(classes) + # size, cap_list, caption, bboxes, labels + + if self.transforms is not None: + image, target = self.transforms(image, target) + # Check that transforms does not change the identity of target['labels']. + if len(target["labels"]) > 0: + assert target["labels"][0] == target["labels_uncropped"][0] + print( + "Asserted that transforms does not change the identity of target['labels']." + ) + + return image, target + + def __len__(self) -> int: + return len(self.metas) + + +def make_coco_transforms(image_set, fix_size=False, strong_aug=False, args=None): + normalize = T.Compose( + [T.ToTensor(), T.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])] + ) + + # config the params for data aug + scales = [480, 512, 544, 576, 608, 640, 672, 704, 736, 768, 800] + max_size = 1333 + scales2_resize = [400, 500, 600] + scales2_crop = [384, 600] + + # update args from config files + scales = getattr(args, "data_aug_scales", scales) + max_size = getattr(args, "data_aug_max_size", max_size) + scales2_resize = getattr(args, "data_aug_scales2_resize", scales2_resize) + scales2_crop = getattr(args, "data_aug_scales2_crop", scales2_crop) + + # resize them + data_aug_scale_overlap = getattr(args, "data_aug_scale_overlap", None) + if data_aug_scale_overlap is not None and data_aug_scale_overlap > 0: + data_aug_scale_overlap = float(data_aug_scale_overlap) + scales = [int(i * data_aug_scale_overlap) for i in scales] + max_size = int(max_size * data_aug_scale_overlap) + scales2_resize = [int(i * data_aug_scale_overlap) for i in scales2_resize] + scales2_crop = [int(i * data_aug_scale_overlap) for i in scales2_crop] + + # datadict_for_print = { + # 'scales': scales, + # 'max_size': max_size, + # 'scales2_resize': scales2_resize, + # 'scales2_crop': scales2_crop + # } + # print("data_aug_params:", json.dumps(datadict_for_print, indent=2)) + + if image_set == "train": + if fix_size: + return T.Compose( + [ + T.RandomHorizontalFlip(), + T.RandomResize([(max_size, max(scales))]), + normalize, + ] + ) + + if strong_aug: + import datasets.sltransform as SLT + + return T.Compose( + [ + T.RandomHorizontalFlip(), + T.RandomSelect( + T.RandomResize(scales, max_size=max_size), + T.Compose( + [ + T.RandomResize(scales2_resize), + T.RandomSizeCrop(*scales2_crop), + T.RandomResize(scales, max_size=max_size), + ] + ), + ), + SLT.RandomSelectMulti( + [ + SLT.RandomCrop(), + SLT.LightingNoise(), + SLT.AdjustBrightness(2), + SLT.AdjustContrast(2), + ] + ), + normalize, + ] + ) + + return T.Compose( + [ + T.RandomHorizontalFlip(), + T.RandomSelect( + T.RandomResize(scales, max_size=max_size), + T.Compose( + [ + T.RandomResize(scales2_resize), + T.RandomSizeCrop(*scales2_crop), + T.RandomResize(scales, max_size=max_size), + ] + ), + ), + normalize, + ] + ) + + if image_set in ["val", "eval_debug", "train_reg", "test"]: + if os.environ.get("GFLOPS_DEBUG_SHILONG", False) == "INFO": + print("Under debug mode for flops calculation only!!!!!!!!!!!!!!!!") + return T.Compose( + [ + T.ResizeDebug((1280, 800)), + normalize, + ] + ) + + return T.Compose( + [ + T.RandomResize([max(scales)], max_size=max_size), + normalize, + ] + ) + + raise ValueError(f"unknown {image_set}") + + +def build_odvg(image_set, args, datasetinfo): + img_folder = datasetinfo["root"] + ann_file = datasetinfo["anno"] + label_map = datasetinfo["label_map"] if "label_map" in datasetinfo else None + try: + strong_aug = args.strong_aug + except: + strong_aug = False + print(img_folder, ann_file, label_map) + dataset = ODVGDataset( + img_folder, + ann_file, + label_map, + max_labels=args.max_labels, + transforms=make_coco_transforms( + image_set, fix_size=args.fix_size, strong_aug=strong_aug, args=args + ), + ) + return dataset + + +if __name__ == "__main__": + dataset_vg = ODVGDataset( + "path/GRIT-20M/data/", + "path/GRIT-20M/anno/grit_odvg_10k.jsonl", + ) + print(len(dataset_vg)) + data = dataset_vg[random.randint(0, 100)] + print(data) + dataset_od = ODVGDataset( + "pathl/V3Det/", + "path/V3Det/annotations/v3det_2023_v1_all_odvg.jsonl", + "path/V3Det/annotations/v3det_label_map.json", + ) + print(len(dataset_od)) + data = dataset_od[random.randint(0, 100)] + print(data) diff --git a/datasets/panoptic_eval.py b/datasets/panoptic_eval.py new file mode 100644 index 0000000000000000000000000000000000000000..9cb4f83409046a5c2a87643ee005e52a440aae74 --- /dev/null +++ b/datasets/panoptic_eval.py @@ -0,0 +1,44 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved +import json +import os + +import util.misc as utils + +try: + from panopticapi.evaluation import pq_compute +except ImportError: + pass + + +class PanopticEvaluator(object): + def __init__(self, ann_file, ann_folder, output_dir="panoptic_eval"): + self.gt_json = ann_file + self.gt_folder = ann_folder + if utils.is_main_process(): + if not os.path.exists(output_dir): + os.mkdir(output_dir) + self.output_dir = output_dir + self.predictions = [] + + def update(self, predictions): + for p in predictions: + with open(os.path.join(self.output_dir, p["file_name"]), "wb") as f: + f.write(p.pop("png_string")) + + self.predictions += predictions + + def synchronize_between_processes(self): + all_predictions = utils.all_gather(self.predictions) + merged_predictions = [] + for p in all_predictions: + merged_predictions += p + self.predictions = merged_predictions + + def summarize(self): + if utils.is_main_process(): + json_data = {"annotations": self.predictions} + predictions_json = os.path.join(self.output_dir, "predictions.json") + with open(predictions_json, "w") as f: + f.write(json.dumps(json_data)) + return pq_compute(self.gt_json, predictions_json, gt_folder=self.gt_folder, pred_folder=self.output_dir) + return None diff --git a/datasets/random_crop.py b/datasets/random_crop.py new file mode 100644 index 0000000000000000000000000000000000000000..a23d936048e073423dd176f76607e9fc2b35bb48 --- /dev/null +++ b/datasets/random_crop.py @@ -0,0 +1,135 @@ +import PIL #version 1.2.0 +import torch +import os +import torchvision.transforms.functional as F +import numpy as np +import random + + +def intersect(boxes1, boxes2): + ''' + Find intersection of every box combination between two sets of box + boxes1: bounding boxes 1, a tensor of dimensions (n1, 4) + boxes2: bounding boxes 2, a tensor of dimensions (n2, 4) + + Out: Intersection each of boxes1 with respect to each of boxes2, + a tensor of dimensions (n1, n2) + ''' + n1 = boxes1.size(0) + n2 = boxes2.size(0) + max_xy = torch.min(boxes1[:, 2:].unsqueeze(1).expand(n1, n2, 2), + boxes2[:, 2:].unsqueeze(0).expand(n1, n2, 2)) + + min_xy = torch.max(boxes1[:, :2].unsqueeze(1).expand(n1, n2, 2), + boxes2[:, :2].unsqueeze(0).expand(n1, n2, 2)) + inter = torch.clamp(max_xy - min_xy , min=0) # (n1, n2, 2) + return inter[:, :, 0] * inter[:, :, 1] #(n1, n2) +def find_IoU(boxes1, boxes2): + ''' + Find IoU between every boxes set of boxes + boxes1: a tensor of dimensions (n1, 4) (left, top, right , bottom) + boxes2: a tensor of dimensions (n2, 4) + + Out: IoU each of boxes1 with respect to each of boxes2, a tensor of + dimensions (n1, n2) + + Formula: + (box1 ∩ box2) / (box1 u box2) = (box1 ∩ box2) / (area(box1) + area(box2) - (box1 ∩ box2 )) + ''' + inter = intersect(boxes1, boxes2) + area_boxes1 = (boxes1[:, 2] - boxes1[:, 0]) * (boxes1[:, 3] - boxes1[:, 1]) + area_boxes2 = (boxes2[:, 2] - boxes2[:, 0]) * (boxes2[:, 3] - boxes2[:, 1]) + + area_boxes1 = area_boxes1.unsqueeze(1).expand_as(inter) #(n1, n2) + area_boxes2 = area_boxes2.unsqueeze(0).expand_as(inter) #(n1, n2) + union = (area_boxes1 + area_boxes2 - inter) + return inter / union + + +def random_crop(image, boxes, labels, difficulties=None): + ''' + image: A PIL image + boxes: Bounding boxes, a tensor of dimensions (#objects, 4) + labels: labels of object, a tensor of dimensions (#objects) + difficulties: difficulties of detect object, a tensor of dimensions (#objects) + + Out: cropped image , new boxes, new labels, new difficulties + ''' + if type(image) == PIL.Image.Image: + image = F.to_tensor(image) + original_h = image.size(1) + original_w = image.size(2) + + while True: + mode = random.choice([0.1, 0.3, 0.5, 0.9, None]) + + if mode is None: + return F.to_pil_image(image), boxes, labels, difficulties + + new_image = image + new_boxes = boxes + new_difficulties = difficulties + new_labels = labels + for _ in range(50): + # Crop dimensions: [0.3, 1] of original dimensions + new_h = random.uniform(0.3*original_h, original_h) + new_w = random.uniform(0.3*original_w, original_w) + + # Aspect ratio constraint b/t .5 & 2 + if new_h/new_w < 0.5 or new_h/new_w > 2: + continue + + #Crop coordinate + left = random.uniform(0, original_w - new_w) + right = left + new_w + top = random.uniform(0, original_h - new_h) + bottom = top + new_h + crop = torch.FloatTensor([int(left), int(top), int(right), int(bottom)]) + + # Calculate IoU between the crop and the bounding boxes + overlap = find_IoU(crop.unsqueeze(0), boxes) #(1, #objects) + overlap = overlap.squeeze(0) + + # If not a single bounding box has a IoU of greater than the minimum, try again + if overlap.shape[0] == 0: + continue + if overlap.max().item() < mode: + continue + + #Crop + new_image = image[:, int(top):int(bottom), int(left):int(right)] #(3, new_h, new_w) + + #Center of bounding boxes + center_bb = (boxes[:, :2] + boxes[:, 2:])/2.0 + + #Find bounding box has been had center in crop + center_in_crop = (center_bb[:, 0] >left) * (center_bb[:, 0] < right + ) *(center_bb[:, 1] > top) * (center_bb[:, 1] < bottom) #( #objects) + + if not center_in_crop.any(): + continue + + #take matching bounding box + new_boxes = boxes[center_in_crop, :] + + #take matching labels + new_labels = labels[center_in_crop] + + #take matching difficulities + if difficulties is not None: + new_difficulties = difficulties[center_in_crop] + else: + new_difficulties = None + + #Use the box left and top corner or the crop's + new_boxes[:, :2] = torch.max(new_boxes[:, :2], crop[:2]) + + #adjust to crop + new_boxes[:, :2] -= crop[:2] + + new_boxes[:, 2:] = torch.min(new_boxes[:, 2:],crop[2:]) + + #adjust to crop + new_boxes[:, 2:] -= crop[:2] + + return F.to_pil_image(new_image), new_boxes, new_labels, new_difficulties \ No newline at end of file diff --git a/datasets/sltransform.py b/datasets/sltransform.py new file mode 100644 index 0000000000000000000000000000000000000000..7d9d10bc20e159f59e46912057714c527a08d6cd --- /dev/null +++ b/datasets/sltransform.py @@ -0,0 +1,247 @@ +# modified from https://github.com/anhtuan85/Data-Augmentation-for-Object-Detection/blob/master/augmentation.ipynb + +import PIL #version 1.2.0 +from PIL import Image #version 6.1.0 +import torch +import os +import torchvision.transforms.functional as F +import numpy as np +import random + +from .random_crop import random_crop +from util.box_ops import box_cxcywh_to_xyxy, box_xyxy_to_cxcywh + +class AdjustContrast: + def __init__(self, contrast_factor): + self.contrast_factor = contrast_factor + + def __call__(self, img, target): + """ + img (PIL Image or Tensor): Image to be adjusted. + """ + _contrast_factor = ((random.random() + 1.0) / 2.0) * self.contrast_factor + img = F.adjust_contrast(img, _contrast_factor) + return img, target + +class AdjustBrightness: + def __init__(self, brightness_factor): + self.brightness_factor = brightness_factor + + def __call__(self, img, target): + """ + img (PIL Image or Tensor): Image to be adjusted. + """ + _brightness_factor = ((random.random() + 1.0) / 2.0) * self.brightness_factor + img = F.adjust_brightness(img, _brightness_factor) + return img, target + +def lighting_noise(image): + ''' + color channel swap in image + image: A PIL image + ''' + new_image = image + perms = ((0, 1, 2), (0, 2, 1), (1, 0, 2), + (1, 2, 0), (2, 0, 1), (2, 1, 0)) + swap = perms[random.randint(0, len(perms)- 1)] + new_image = F.to_tensor(new_image) + new_image = new_image[swap, :, :] + new_image = F.to_pil_image(new_image) + return new_image + +class LightingNoise: + def __init__(self) -> None: + pass + + def __call__(self, img, target): + return lighting_noise(img), target + + +def rotate(image, boxes, angle): + ''' + Rotate image and bounding box + image: A Pil image (w, h) + boxes: A tensors of dimensions (#objects, 4) + + Out: rotated image (w, h), rotated boxes + ''' + new_image = image.copy() + new_boxes = boxes.clone() + + #Rotate image, expand = True + w = image.width + h = image.height + cx = w/2 + cy = h/2 + new_image = new_image.rotate(angle, expand=True) + angle = np.radians(angle) + alpha = np.cos(angle) + beta = np.sin(angle) + #Get affine matrix + AffineMatrix = torch.tensor([[alpha, beta, (1-alpha)*cx - beta*cy], + [-beta, alpha, beta*cx + (1-alpha)*cy]]) + + #Rotation boxes + box_width = (boxes[:,2] - boxes[:,0]).reshape(-1,1) + box_height = (boxes[:,3] - boxes[:,1]).reshape(-1,1) + + #Get corners for boxes + x1 = boxes[:,0].reshape(-1,1) + y1 = boxes[:,1].reshape(-1,1) + + x2 = x1 + box_width + y2 = y1 + + x3 = x1 + y3 = y1 + box_height + + x4 = boxes[:,2].reshape(-1,1) + y4 = boxes[:,3].reshape(-1,1) + + corners = torch.stack((x1,y1,x2,y2,x3,y3,x4,y4), dim= 1) + # corners.reshape(-1, 8) #Tensors of dimensions (#objects, 8) + corners = corners.reshape(-1,2) #Tensors of dimension (4* #objects, 2) + corners = torch.cat((corners, torch.ones(corners.shape[0], 1)), dim= 1) #(Tensors of dimension (4* #objects, 3)) + + cos = np.abs(AffineMatrix[0, 0]) + sin = np.abs(AffineMatrix[0, 1]) + + nW = int((h * sin) + (w * cos)) + nH = int((h * cos) + (w * sin)) + AffineMatrix[0, 2] += (nW / 2) - cx + AffineMatrix[1, 2] += (nH / 2) - cy + + + #Apply affine transform + rotate_corners = torch.mm(AffineMatrix, corners.t().to(torch.float64)).t() + rotate_corners = rotate_corners.reshape(-1,8) + + x_corners = rotate_corners[:,[0,2,4,6]] + y_corners = rotate_corners[:,[1,3,5,7]] + + #Get (x_min, y_min, x_max, y_max) + x_min, _ = torch.min(x_corners, dim= 1) + x_min = x_min.reshape(-1, 1) + y_min, _ = torch.min(y_corners, dim= 1) + y_min = y_min.reshape(-1, 1) + x_max, _ = torch.max(x_corners, dim= 1) + x_max = x_max.reshape(-1, 1) + y_max, _ = torch.max(y_corners, dim= 1) + y_max = y_max.reshape(-1, 1) + + new_boxes = torch.cat((x_min, y_min, x_max, y_max), dim= 1) + + scale_x = new_image.width / w + scale_y = new_image.height / h + + #Resize new image to (w, h) + + new_image = new_image.resize((w, h)) + + #Resize boxes + new_boxes /= torch.Tensor([scale_x, scale_y, scale_x, scale_y]) + new_boxes[:, 0] = torch.clamp(new_boxes[:, 0], 0, w) + new_boxes[:, 1] = torch.clamp(new_boxes[:, 1], 0, h) + new_boxes[:, 2] = torch.clamp(new_boxes[:, 2], 0, w) + new_boxes[:, 3] = torch.clamp(new_boxes[:, 3], 0, h) + return new_image, new_boxes + +# def convert_xywh_to_xyxy(boxes: torch.Tensor): +# _boxes = boxes.clone() +# box_xy = _boxes[:, :2] +# box_wh = _boxes[:, 2:] +# box_x1y1 = box_xy - box_wh/2 +# box_x2y2 = box_xy + box_wh/2 +# box_xyxy = torch.cat((box_x1y1, box_x2y2), dim=-1) +# return box_xyxy + +class Rotate: + def __init__(self, angle=10) -> None: + self.angle = angle + + def __call__(self, img, target): + w,h = img.size + whwh = torch.Tensor([w, h, w, h]) + boxes_xyxy = box_cxcywh_to_xyxy(target['boxes']) * whwh + img, boxes_new = rotate(img, boxes_xyxy, self.angle) + target['boxes'] = box_xyxy_to_cxcywh(boxes_new).to(boxes_xyxy.dtype) / (whwh + 1e-3) + return img, target + + +class RandomCrop: + def __init__(self) -> None: + pass + + def __call__(self, img, target): + w,h = img.size + try: + boxes_xyxy = target['boxes'] + labels = target['labels'] + img, new_boxes, new_labels, _ = random_crop(img, boxes_xyxy, labels) + target['boxes'] = new_boxes + target['labels'] = new_labels + except Exception as e: + pass + return img, target + + +class RandomCropDebug: + def __init__(self) -> None: + pass + + def __call__(self, img, target): + boxes_xyxy = target['boxes'].clone() + labels = target['labels'].clone() + img, new_boxes, new_labels, _ = random_crop(img, boxes_xyxy, labels) + target['boxes'] = new_boxes + target['labels'] = new_labels + + + return img, target + +class RandomSelectMulti(object): + """ + Randomly selects between transforms1 and transforms2, + """ + def __init__(self, transformslist, p=-1): + self.transformslist = transformslist + self.p = p + assert p == -1 + + def __call__(self, img, target): + if self.p == -1: + return random.choice(self.transformslist)(img, target) + + +class Albumentations: + def __init__(self): + import albumentations as A + self.transform = A.Compose([ + A.Blur(p=0.01), + A.MedianBlur(p=0.01), + A.ToGray(p=0.01), + A.CLAHE(p=0.01), + A.RandomBrightnessContrast(p=0.005), + A.RandomGamma(p=0.005), + A.ImageCompression(quality_lower=75, p=0.005)], + bbox_params=A.BboxParams(format='pascal_voc', label_fields=['class_labels'])) + + def __call__(self, img, target, p=1.0): + """ + Input: + target['boxes']: xyxy, unnormalized data. + + """ + boxes_raw = target['boxes'] + labels_raw = target['labels'] + img_np = np.array(img) + if self.transform and random.random() < p: + new_res = self.transform(image=img_np, bboxes=boxes_raw, class_labels=labels_raw) # transformed + boxes_new = torch.Tensor(new_res['bboxes']).to(boxes_raw.dtype).reshape_as(boxes_raw) + img_np = new_res['image'] + labels_new = torch.Tensor(new_res['class_labels']).to(labels_raw.dtype) + img_new = Image.fromarray(img_np) + target['boxes'] = boxes_new + target['labels'] = labels_new + + return img_new, target \ No newline at end of file diff --git a/datasets/transforms.py b/datasets/transforms.py new file mode 100644 index 0000000000000000000000000000000000000000..e4ac7ac8afde6e7a3d3d9746dbd87d2bde2e6879 --- /dev/null +++ b/datasets/transforms.py @@ -0,0 +1,338 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved +""" +Transforms and data augmentation for both image + bbox. +""" +import random + +import PIL +import torch +import torchvision.transforms as T +import torchvision.transforms.functional as F + +from util.box_ops import box_xyxy_to_cxcywh +from util.misc import interpolate + + +def crop(image, target, region): + cropped_image = F.crop(image, *region) + + target = target.copy() + i, j, h, w = region + + # should we do something wrt the original size? + target["size"] = torch.tensor([h, w]) + + fields = ["labels", "area"] + + # Crop exemplars. + exemplars = target["exemplars"] + max_size = torch.as_tensor([w, h], dtype=torch.float32) + # Shift exemplars to cropped region. + cropped_exemplars = exemplars - torch.as_tensor([j, i, j, i]) + # Correct exemplar regions that go past new image boundary (too far right). + cropped_exemplars = torch.min(cropped_exemplars.reshape(-1, 2, 2), max_size) + # Correct exemplar regions that go past new image boundary (too far left). + cropped_exemplars = cropped_exemplars.clamp(min=0) + # Get new exemplar areas. + area_exemplars = (cropped_exemplars[:, 1, :] - cropped_exemplars[:, 0, :]).prod( + dim=1 + ) + # Update [target] with cropped exemplars. + target["exemplars"] = cropped_exemplars.reshape(-1, 4) + if "boxes" in target: + boxes = target["boxes"] + max_size = torch.as_tensor([w, h], dtype=torch.float32) + cropped_boxes = boxes - torch.as_tensor([j, i, j, i]) + cropped_boxes = torch.min(cropped_boxes.reshape(-1, 2, 2), max_size) + cropped_boxes = cropped_boxes.clamp(min=0) + area = (cropped_boxes[:, 1, :] - cropped_boxes[:, 0, :]).prod(dim=1) + target["boxes"] = cropped_boxes.reshape(-1, 4) + target["area"] = area + fields.append("boxes") + + if "masks" in target: + # FIXME should we update the area here if there are no boxes? + target["masks"] = target["masks"][:, i : i + h, j : j + w] + fields.append("masks") + + # Remove exemplars that have zero area (due to cropping). + keep = area_exemplars > 0 + target["exemplars"] = target["exemplars"][keep, :] + + # remove elements for which the boxes or masks that have zero area + if "boxes" in target or "masks" in target: + # favor boxes selection when defining which elements to keep + # this is compatible with previous implementation + if "boxes" in target: + cropped_boxes = target["boxes"].reshape(-1, 2, 2) + keep = torch.all(cropped_boxes[:, 1, :] > cropped_boxes[:, 0, :], dim=1) + else: + keep = target["masks"].flatten(1).any(1) + + for field in fields: + target[field] = target[field][keep] + + return cropped_image, target + + +def hflip(image, target): + flipped_image = F.hflip(image) + + w, h = image.size + + target = target.copy() + + exemplars = target["exemplars"] + # Flip image across x-axis. + exemplars = exemplars[:, [2, 1, 0, 3]] * torch.as_tensor([-1, 1, -1, 1]) + # Shift flipped image to (0, 0). + exemplars = exemplars + torch.as_tensor([w, 0, w, 0]) + # Update [target] with horizontally flipped exemplars. + target["exemplars"] = exemplars + + if "boxes" in target: + boxes = target["boxes"] + boxes = boxes[:, [2, 1, 0, 3]] * torch.as_tensor( + [-1, 1, -1, 1] + ) + torch.as_tensor([w, 0, w, 0]) + target["boxes"] = boxes + + if "masks" in target: + target["masks"] = target["masks"].flip(-1) + + return flipped_image, target + + +def resize(image, target, size, max_size=None): + # size can be min_size (scalar) or (w, h) tuple + + def get_size_with_aspect_ratio(image_size, size, max_size=None): + w, h = image_size + if max_size is not None: + min_original_size = float(min((w, h))) + max_original_size = float(max((w, h))) + if max_original_size / min_original_size * size > max_size: + size = int(round(max_size * min_original_size / max_original_size)) + + if (w <= h and w == size) or (h <= w and h == size): + return (h, w) + + if w < h: + ow = size + oh = int(size * h / w) + else: + oh = size + ow = int(size * w / h) + + return (oh, ow) + + def get_size(image_size, size, max_size=None): + if isinstance(size, (list, tuple)): + return size[::-1] + else: + return get_size_with_aspect_ratio(image_size, size, max_size) + + try: + size = get_size(image.size, size, max_size) + except: + size = get_size((image.shape[-1], image.shape[-2]), size, max_size) + rescaled_image = F.resize(image, size) + + if target is None: + return rescaled_image, None + + ratios = tuple( + float(s) / float(s_orig) for s, s_orig in zip(rescaled_image.size, image.size) + ) + ratio_width, ratio_height = ratios + + target = target.copy() + + # Rescale exemplars. + exemplars = target["exemplars"] + if exemplars.shape[-1] == 4: + scaled_exemplars = exemplars * torch.as_tensor( + [ratio_width, ratio_height, ratio_width, ratio_height] + ) + else: + scaled_exemplars = exemplars + + target["exemplars"] = scaled_exemplars + + if "boxes" in target: + boxes = target["boxes"] + scaled_boxes = boxes * torch.as_tensor( + [ratio_width, ratio_height, ratio_width, ratio_height] + ) + target["boxes"] = scaled_boxes + + if "area" in target: + area = target["area"] + scaled_area = area * (ratio_width * ratio_height) + target["area"] = scaled_area + + h, w = size + target["size"] = torch.tensor([h, w]) + + if "masks" in target: + target["masks"] = ( + interpolate(target["masks"][:, None].float(), size, mode="nearest")[:, 0] + > 0.5 + ) + + return rescaled_image, target + + +def pad(image, target, padding): + # assumes that we only pad on the bottom right corners + padded_image = F.pad(image, (0, 0, padding[0], padding[1])) + if target is None: + return padded_image, None + target = target.copy() + # should we do something wrt the original size? + target["size"] = torch.tensor(padded_image.size[::-1]) + if "masks" in target: + target["masks"] = torch.nn.functional.pad( + target["masks"], (0, padding[0], 0, padding[1]) + ) + return padded_image, target + + +class ResizeDebug(object): + def __init__(self, size): + self.size = size + + def __call__(self, img, target): + return resize(img, target, self.size) + + +class RandomCrop(object): + def __init__(self, size): + self.size = size + + def __call__(self, img, target): + region = T.RandomCrop.get_params(img, self.size) + return crop(img, target, region) + + +class RandomSizeCrop(object): + def __init__(self, min_size: int, max_size: int): + self.min_size = min_size + self.max_size = max_size + + def __call__(self, img: PIL.Image.Image, target: dict): + w = random.randint(self.min_size, min(img.width, self.max_size)) + h = random.randint(self.min_size, min(img.height, self.max_size)) + region = T.RandomCrop.get_params(img, [h, w]) + return crop(img, target, region) + + +class CenterCrop(object): + def __init__(self, size): + self.size = size + + def __call__(self, img, target): + image_width, image_height = img.size + crop_height, crop_width = self.size + crop_top = int(round((image_height - crop_height) / 2.0)) + crop_left = int(round((image_width - crop_width) / 2.0)) + return crop(img, target, (crop_top, crop_left, crop_height, crop_width)) + + +class RandomHorizontalFlip(object): + def __init__(self, p=0.5): + self.p = p + + def __call__(self, img, target): + if random.random() < self.p: + return hflip(img, target) + return img, target + + +class RandomResize(object): + def __init__(self, sizes, max_size=None): + assert isinstance(sizes, (list, tuple)) + self.sizes = sizes + self.max_size = max_size + + def __call__(self, img, target=None): + size = random.choice(self.sizes) + return resize(img, target, size, self.max_size) + + +class RandomPad(object): + def __init__(self, max_pad): + self.max_pad = max_pad + + def __call__(self, img, target): + pad_x = random.randint(0, self.max_pad) + pad_y = random.randint(0, self.max_pad) + return pad(img, target, (pad_x, pad_y)) + + +class RandomSelect(object): + """ + Randomly selects between transforms1 and transforms2, + with probability p for transforms1 and (1 - p) for transforms2 + """ + + def __init__(self, transforms1, transforms2, p=0.5): + self.transforms1 = transforms1 + self.transforms2 = transforms2 + self.p = p + + def __call__(self, img, target): + if random.random() < self.p: + return self.transforms1(img, target) + return self.transforms2(img, target) + + +class ToTensor(object): + def __call__(self, img, target): + return F.to_tensor(img), target + + +class RandomErasing(object): + def __init__(self, *args, **kwargs): + self.eraser = T.RandomErasing(*args, **kwargs) + + def __call__(self, img, target): + return self.eraser(img), target + + +class Normalize(object): + def __init__(self, mean, std): + self.mean = mean + self.std = std + + def __call__(self, image, target=None): + image = F.normalize(image, mean=self.mean, std=self.std) + if target is None: + return image, None + target = target.copy() + h, w = image.shape[-2:] + # No normalization of exemplars needed, since they are used directly for cropping. + if "boxes" in target: + boxes = target["boxes"] + boxes = box_xyxy_to_cxcywh(boxes) + boxes = boxes / torch.tensor([w, h, w, h], dtype=torch.float32) + target["boxes"] = boxes + return image, target + + +class Compose(object): + def __init__(self, transforms): + self.transforms = transforms + + def __call__(self, image, target): + for t in self.transforms: + image, target = t(image, target) + return image, target + + def __repr__(self): + format_string = self.__class__.__name__ + "(" + for t in self.transforms: + format_string += "\n" + format_string += " {0}".format(t) + format_string += "\n)" + return format_string diff --git a/debug/config_args_all.json b/debug/config_args_all.json new file mode 100644 index 0000000000000000000000000000000000000000..fc96556ec4ce6c395da4237e851e3d1e50122d1a --- /dev/null +++ b/debug/config_args_all.json @@ -0,0 +1,308 @@ +{ + "config_file": "config/cfg_fsc147_vit_b.py", + "options": { + "text_encoder_type": "checkpoints/bert-base-uncased" + }, + "datasets": "config/datasets_fsc147.json", + "remove_difficult": false, + "fix_size": false, + "output_dir": "./debug", + "note": "", + "device": "cuda", + "seed": 42, + "resume": "", + "pretrain_model_path": "checkpoints/groundingdino_swinb_cogcoor.pth", + "finetune_ignore": null, + "start_epoch": 0, + "eval": false, + "num_workers": 8, + "test": false, + "debug": false, + "find_unused_params": false, + "save_results": false, + "save_log": false, + "world_size": 1, + "dist_url": "env://", + "rank": 0, + "local_rank": 0, + "amp": false, + "distributed": false, + "data_aug_scales": [ + 480, + 512, + 544, + 576, + 608, + 640, + 672, + 704, + 736, + 768, + 800 + ], + "data_aug_max_size": 1333, + "data_aug_scales2_resize": [ + 400, + 500, + 600 + ], + "data_aug_scales2_crop": [ + 384, + 600 + ], + "data_aug_scale_overlap": null, + "batch_size": 4, + "modelname": "groundingdino", + "backbone": "swin_B_384_22k", + "position_embedding": "sine", + "pe_temperatureH": 20, + "pe_temperatureW": 20, + "return_interm_indices": [ + 1, + 2, + 3 + ], + "enc_layers": 6, + "dec_layers": 6, + "pre_norm": false, + "dim_feedforward": 2048, + "hidden_dim": 256, + "dropout": 0.0, + "nheads": 8, + "num_queries": 900, + "query_dim": 4, + "num_patterns": 0, + "num_feature_levels": 4, + "enc_n_points": 4, + "dec_n_points": 4, + "two_stage_type": "standard", + "two_stage_bbox_embed_share": false, + "two_stage_class_embed_share": false, + "transformer_activation": "relu", + "dec_pred_bbox_embed_share": true, + "dn_box_noise_scale": 1.0, + "dn_label_noise_ratio": 0.5, + "dn_label_coef": 1.0, + "dn_bbox_coef": 1.0, + "embed_init_tgt": true, + "dn_labelbook_size": 91, + "max_text_len": 256, + "text_encoder_type": "checkpoints/bert-base-uncased", + "use_text_enhancer": true, + "use_fusion_layer": true, + "use_checkpoint": true, + "use_transformer_ckpt": true, + "use_text_cross_attention": true, + "text_dropout": 0.0, + "fusion_dropout": 0.0, + "fusion_droppath": 0.1, + "sub_sentence_present": true, + "max_labels": 90, + "lr": 0.0001, + "backbone_freeze_keywords": null, + "freeze_keywords": [ + "backbone.0", + "bert" + ], + "lr_backbone": 1e-05, + "lr_backbone_names": [ + "backbone.0", + "bert" + ], + "lr_linear_proj_mult": 1e-05, + "lr_linear_proj_names": [ + "ref_point_head", + "sampling_offsets" + ], + "weight_decay": 0.0001, + "param_dict_type": "ddetr_in_mmdet", + "ddetr_lr_param": false, + "epochs": 30, + "lr_drop": 10, + "save_checkpoint_interval": 10, + "clip_max_norm": 0.1, + "onecyclelr": false, + "multi_step_lr": false, + "lr_drop_list": [ + 10, + 20 + ], + "frozen_weights": null, + "dilation": false, + "pdetr3_bbox_embed_diff_each_layer": false, + "pdetr3_refHW": -1, + "random_refpoints_xy": false, + "fix_refpoints_hw": -1, + "dabdetr_yolo_like_anchor_update": false, + "dabdetr_deformable_encoder": false, + "dabdetr_deformable_decoder": false, + "use_deformable_box_attn": false, + "box_attn_type": "roi_align", + "dec_layer_number": null, + "decoder_layer_noise": false, + "dln_xy_noise": 0.2, + "dln_hw_noise": 0.2, + "add_channel_attention": false, + "add_pos_value": false, + "two_stage_pat_embed": 0, + "two_stage_add_query_num": 0, + "two_stage_learn_wh": false, + "two_stage_default_hw": 0.05, + "two_stage_keep_all_tokens": false, + "num_select": 900, + "batch_norm_type": "FrozenBatchNorm2d", + "masks": false, + "aux_loss": true, + "set_cost_class": 5.0, + "set_cost_bbox": 1.0, + "set_cost_giou": 0.0, + "cls_loss_coef": 5.0, + "bbox_loss_coef": 1.0, + "giou_loss_coef": 0.0, + "enc_loss_coef": 1.0, + "interm_loss_coef": 1.0, + "no_interm_box_loss": false, + "mask_loss_coef": 1.0, + "dice_loss_coef": 1.0, + "focal_alpha": 0.25, + "focal_gamma": 2.0, + "decoder_sa_type": "sa", + "matcher_type": "HungarianMatcher", + "decoder_module_seq": [ + "sa", + "ca", + "ffn" + ], + "nms_iou_threshold": -1, + "dec_pred_class_embed_share": true, + "match_unstable_error": true, + "use_detached_boxes_dec_out": false, + "dn_scalar": 100, + "box_threshold": 0.23, + "text_threshold": 0, + "use_coco_eval": false, + "label_list": [ + "alcohol bottle", + "baguette roll", + "ball", + "banana", + "bead", + "bee", + "birthday candle", + "biscuit", + "boat", + "bottle", + "bowl", + "box", + "bread roll", + "brick", + "buffalo", + "bun", + "calamari ring", + "can", + "candle", + "cap", + "car", + "cartridge", + "cassette", + "cement bag", + "cereal", + "chewing gum piece", + "chopstick", + "clam", + "coffee bean", + "coin", + "cotton ball", + "cow", + "crane", + "crayon", + "croissant", + "crow", + "cup", + "cupcake", + "cupcake holder", + "fish", + "gemstone", + "go game piece", + "goat", + "goldfish snack", + "goose", + "ice cream", + "ice cream cone", + "instant noodle", + "jade stone", + "jeans", + "kidney bean", + "kitchen towel", + "lighter", + "lipstick", + "m&m piece", + "macaron", + "match", + "meat skewer", + "mini blind", + "mosaic tile", + "naan bread", + "nail", + "nut", + "onion ring", + "orange", + "pearl", + "pen", + "pencil", + "penguin", + "pepper", + "person", + "pigeon", + "plate", + "polka dot tile", + "potato", + "rice bag", + "roof tile", + "screw", + "shoe", + "spoon", + "spring roll", + "stair", + "stapler pin", + "straw", + "supermarket shelf", + "swan", + "tomato", + "watermelon", + "window", + "zebra" + ], + "val_label_list": [ + "ant", + "bird", + "book", + "bottle cap", + "bullet", + "camel", + "chair", + "chicken wing", + "donut", + "donut holder", + "flamingo", + "flower", + "flower pot", + "grape", + "horse", + "kiwi", + "milk carton", + "oyster", + "oyster shell", + "package of fresh cut fruit", + "peach", + "pill", + "polka dot", + "prawn cracker", + "sausage", + "seagull", + "shallot", + "shirt", + "skateboard", + "toilet paper roll" + ] +} \ No newline at end of file diff --git a/debug/config_args_raw.json b/debug/config_args_raw.json new file mode 100644 index 0000000000000000000000000000000000000000..48ece117d2181456142ff1bb87b96a41437c6596 --- /dev/null +++ b/debug/config_args_raw.json @@ -0,0 +1,30 @@ +{ + "config_file": "config/cfg_fsc147_vit_b.py", + "options": { + "text_encoder_type": "checkpoints/bert-base-uncased" + }, + "datasets": "config/datasets_fsc147.json", + "remove_difficult": false, + "fix_size": false, + "output_dir": "./debug", + "note": "", + "device": "cuda", + "seed": 42, + "resume": "", + "pretrain_model_path": "checkpoints/groundingdino_swinb_cogcoor.pth", + "finetune_ignore": null, + "start_epoch": 0, + "eval": false, + "num_workers": 8, + "test": false, + "debug": false, + "find_unused_params": false, + "save_results": false, + "save_log": false, + "world_size": 1, + "dist_url": "env://", + "rank": 0, + "local_rank": 0, + "amp": false, + "distributed": false +} \ No newline at end of file diff --git a/debug/config_cfg.py b/debug/config_cfg.py new file mode 100644 index 0000000000000000000000000000000000000000..9bc4d4277e9fcf1fc168dcc31bb773e1559efddc --- /dev/null +++ b/debug/config_cfg.py @@ -0,0 +1,140 @@ +data_aug_scales = [480, 512, 544, 576, 608, 640, 672, 704, 736, 768, 800] +data_aug_max_size = 1333 +data_aug_scales2_resize = [400, 500, 600] +data_aug_scales2_crop = [384, 600] +data_aug_scale_overlap = None +batch_size = 4 +modelname = 'groundingdino' +backbone = 'swin_B_384_22k' +position_embedding = 'sine' +pe_temperatureH = 20 +pe_temperatureW = 20 +return_interm_indices = [1, 2, 3] +enc_layers = 6 +dec_layers = 6 +pre_norm = False +dim_feedforward = 2048 +hidden_dim = 256 +dropout = 0.0 +nheads = 8 +num_queries = 900 +query_dim = 4 +num_patterns = 0 +num_feature_levels = 4 +enc_n_points = 4 +dec_n_points = 4 +two_stage_type = 'standard' +two_stage_bbox_embed_share = False +two_stage_class_embed_share = False +transformer_activation = 'relu' +dec_pred_bbox_embed_share = True +dn_box_noise_scale = 1.0 +dn_label_noise_ratio = 0.5 +dn_label_coef = 1.0 +dn_bbox_coef = 1.0 +embed_init_tgt = True +dn_labelbook_size = 91 +max_text_len = 256 +text_encoder_type = 'checkpoints/bert-base-uncased' +use_text_enhancer = True +use_fusion_layer = True +use_checkpoint = True +use_transformer_ckpt = True +use_text_cross_attention = True +text_dropout = 0.0 +fusion_dropout = 0.0 +fusion_droppath = 0.1 +sub_sentence_present = True +max_labels = 90 +lr = 0.0001 +backbone_freeze_keywords = None +freeze_keywords = ['backbone.0', 'bert'] +lr_backbone = 1e-05 +lr_backbone_names = ['backbone.0', 'bert'] +lr_linear_proj_mult = 1e-05 +lr_linear_proj_names = ['ref_point_head', 'sampling_offsets'] +weight_decay = 0.0001 +param_dict_type = 'ddetr_in_mmdet' +ddetr_lr_param = False +epochs = 30 +lr_drop = 10 +save_checkpoint_interval = 10 +clip_max_norm = 0.1 +onecyclelr = False +multi_step_lr = False +lr_drop_list = [10, 20] +frozen_weights = None +dilation = False +pdetr3_bbox_embed_diff_each_layer = False +pdetr3_refHW = -1 +random_refpoints_xy = False +fix_refpoints_hw = -1 +dabdetr_yolo_like_anchor_update = False +dabdetr_deformable_encoder = False +dabdetr_deformable_decoder = False +use_deformable_box_attn = False +box_attn_type = 'roi_align' +dec_layer_number = None +decoder_layer_noise = False +dln_xy_noise = 0.2 +dln_hw_noise = 0.2 +add_channel_attention = False +add_pos_value = False +two_stage_pat_embed = 0 +two_stage_add_query_num = 0 +two_stage_learn_wh = False +two_stage_default_hw = 0.05 +two_stage_keep_all_tokens = False +num_select = 900 +batch_norm_type = 'FrozenBatchNorm2d' +masks = False +aux_loss = True +set_cost_class = 5.0 +set_cost_bbox = 1.0 +set_cost_giou = 0.0 +cls_loss_coef = 5.0 +bbox_loss_coef = 1.0 +giou_loss_coef = 0.0 +enc_loss_coef = 1.0 +interm_loss_coef = 1.0 +no_interm_box_loss = False +mask_loss_coef = 1.0 +dice_loss_coef = 1.0 +focal_alpha = 0.25 +focal_gamma = 2.0 +decoder_sa_type = 'sa' +matcher_type = 'HungarianMatcher' +decoder_module_seq = ['sa', 'ca', 'ffn'] +nms_iou_threshold = -1 +dec_pred_class_embed_share = True +match_unstable_error = True +use_detached_boxes_dec_out = False +dn_scalar = 100 +box_threshold = 0.23 +text_threshold = 0 +use_coco_eval = False +label_list = [ + 'alcohol bottle', 'baguette roll', 'ball', 'banana', 'bead', 'bee', + 'birthday candle', 'biscuit', 'boat', 'bottle', 'bowl', 'box', + 'bread roll', 'brick', 'buffalo', 'bun', 'calamari ring', 'can', 'candle', + 'cap', 'car', 'cartridge', 'cassette', 'cement bag', 'cereal', + 'chewing gum piece', 'chopstick', 'clam', 'coffee bean', 'coin', + 'cotton ball', 'cow', 'crane', 'crayon', 'croissant', 'crow', 'cup', + 'cupcake', 'cupcake holder', 'fish', 'gemstone', 'go game piece', 'goat', + 'goldfish snack', 'goose', 'ice cream', 'ice cream cone', 'instant noodle', + 'jade stone', 'jeans', 'kidney bean', 'kitchen towel', 'lighter', + 'lipstick', 'm&m piece', 'macaron', 'match', 'meat skewer', 'mini blind', + 'mosaic tile', 'naan bread', 'nail', 'nut', 'onion ring', 'orange', + 'pearl', 'pen', 'pencil', 'penguin', 'pepper', 'person', 'pigeon', 'plate', + 'polka dot tile', 'potato', 'rice bag', 'roof tile', 'screw', 'shoe', + 'spoon', 'spring roll', 'stair', 'stapler pin', 'straw', + 'supermarket shelf', 'swan', 'tomato', 'watermelon', 'window', 'zebra' +] +val_label_list = [ + 'ant', 'bird', 'book', 'bottle cap', 'bullet', 'camel', 'chair', + 'chicken wing', 'donut', 'donut holder', 'flamingo', 'flower', + 'flower pot', 'grape', 'horse', 'kiwi', 'milk carton', 'oyster', + 'oyster shell', 'package of fresh cut fruit', 'peach', 'pill', 'polka dot', + 'prawn cracker', 'sausage', 'seagull', 'shallot', 'shirt', 'skateboard', + 'toilet paper roll' +] diff --git a/debug/info.txt b/debug/info.txt new file mode 100644 index 0000000000000000000000000000000000000000..f9d2eba302d407d8973e94b1b2cc01d1781c4581 --- /dev/null +++ b/debug/info.txt @@ -0,0 +1,1604 @@ +INFO  2024-03-29 17:46:26,830 | git: + sha: N/A, status: clean, branch: N/A + +INFO  2024-03-29 17:46:26,830 | Command: main.py --output_dir ./debug -c config/cfg_fsc147_vit_b.py --datasets config/datasets_fsc147.json --pretrain_model_path checkpoints/groundingdino_swinb_cogcoor.pth --options text_encoder_type=checkpoints/bert-base-uncased +INFO  2024-03-29 17:46:26,831 | Full config saved to ./debug/config_args_all.json +INFO  2024-03-29 17:46:26,831 | world size: 1 +INFO  2024-03-29 17:46:26,831 | rank: 0 +INFO  2024-03-29 17:46:26,831 | local_rank: 0 +INFO  2024-03-29 17:46:26,831 | args: Namespace(config_file='config/cfg_fsc147_vit_b.py', options={'text_encoder_type': 'checkpoints/bert-base-uncased'}, datasets='config/datasets_fsc147.json', remove_difficult=False, fix_size=False, output_dir='./debug', note='', device='cuda', seed=42, resume='', pretrain_model_path='checkpoints/groundingdino_swinb_cogcoor.pth', finetune_ignore=None, start_epoch=0, eval=False, num_workers=8, test=False, debug=False, find_unused_params=False, save_results=False, save_log=False, world_size=1, dist_url='env://', rank=0, local_rank=0, amp=False, distributed=False, data_aug_scales=[480, 512, 544, 576, 608, 640, 672, 704, 736, 768, 800], data_aug_max_size=1333, data_aug_scales2_resize=[400, 500, 600], data_aug_scales2_crop=[384, 600], data_aug_scale_overlap=None, batch_size=4, modelname='groundingdino', backbone='swin_B_384_22k', position_embedding='sine', pe_temperatureH=20, pe_temperatureW=20, return_interm_indices=[1, 2, 3], enc_layers=6, dec_layers=6, pre_norm=False, dim_feedforward=2048, hidden_dim=256, dropout=0.0, nheads=8, num_queries=900, query_dim=4, num_patterns=0, num_feature_levels=4, enc_n_points=4, dec_n_points=4, two_stage_type='standard', two_stage_bbox_embed_share=False, two_stage_class_embed_share=False, transformer_activation='relu', dec_pred_bbox_embed_share=True, dn_box_noise_scale=1.0, dn_label_noise_ratio=0.5, dn_label_coef=1.0, dn_bbox_coef=1.0, embed_init_tgt=True, dn_labelbook_size=91, max_text_len=256, text_encoder_type='checkpoints/bert-base-uncased', use_text_enhancer=True, use_fusion_layer=True, use_checkpoint=True, use_transformer_ckpt=True, use_text_cross_attention=True, text_dropout=0.0, fusion_dropout=0.0, fusion_droppath=0.1, sub_sentence_present=True, max_labels=90, lr=0.0001, backbone_freeze_keywords=None, freeze_keywords=['backbone.0', 'bert'], lr_backbone=1e-05, lr_backbone_names=['backbone.0', 'bert'], lr_linear_proj_mult=1e-05, lr_linear_proj_names=['ref_point_head', 'sampling_offsets'], weight_decay=0.0001, param_dict_type='ddetr_in_mmdet', ddetr_lr_param=False, epochs=30, lr_drop=10, save_checkpoint_interval=10, clip_max_norm=0.1, onecyclelr=False, multi_step_lr=False, lr_drop_list=[10, 20], frozen_weights=None, dilation=False, pdetr3_bbox_embed_diff_each_layer=False, pdetr3_refHW=-1, random_refpoints_xy=False, fix_refpoints_hw=-1, dabdetr_yolo_like_anchor_update=False, dabdetr_deformable_encoder=False, dabdetr_deformable_decoder=False, use_deformable_box_attn=False, box_attn_type='roi_align', dec_layer_number=None, decoder_layer_noise=False, dln_xy_noise=0.2, dln_hw_noise=0.2, add_channel_attention=False, add_pos_value=False, two_stage_pat_embed=0, two_stage_add_query_num=0, two_stage_learn_wh=False, two_stage_default_hw=0.05, two_stage_keep_all_tokens=False, num_select=900, batch_norm_type='FrozenBatchNorm2d', masks=False, aux_loss=True, set_cost_class=5.0, set_cost_bbox=1.0, set_cost_giou=0.0, cls_loss_coef=5.0, bbox_loss_coef=1.0, giou_loss_coef=0.0, enc_loss_coef=1.0, interm_loss_coef=1.0, no_interm_box_loss=False, mask_loss_coef=1.0, dice_loss_coef=1.0, focal_alpha=0.25, focal_gamma=2.0, decoder_sa_type='sa', matcher_type='HungarianMatcher', decoder_module_seq=['sa', 'ca', 'ffn'], nms_iou_threshold=-1, dec_pred_class_embed_share=True, match_unstable_error=True, use_detached_boxes_dec_out=False, dn_scalar=100, box_threshold=0.23, text_threshold=0, use_coco_eval=False, label_list=['alcohol bottle', 'baguette roll', 'ball', 'banana', 'bead', 'bee', 'birthday candle', 'biscuit', 'boat', 'bottle', 'bowl', 'box', 'bread roll', 'brick', 'buffalo', 'bun', 'calamari ring', 'can', 'candle', 'cap', 'car', 'cartridge', 'cassette', 'cement bag', 'cereal', 'chewing gum piece', 'chopstick', 'clam', 'coffee bean', 'coin', 'cotton ball', 'cow', 'crane', 'crayon', 'croissant', 'crow', 'cup', 'cupcake', 'cupcake holder', 'fish', 'gemstone', 'go game piece', 'goat', 'goldfish snack', 'goose', 'ice cream', 'ice cream cone', 'instant noodle', 'jade stone', 'jeans', 'kidney bean', 'kitchen towel', 'lighter', 'lipstick', 'm&m piece', 'macaron', 'match', 'meat skewer', 'mini blind', 'mosaic tile', 'naan bread', 'nail', 'nut', 'onion ring', 'orange', 'pearl', 'pen', 'pencil', 'penguin', 'pepper', 'person', 'pigeon', 'plate', 'polka dot tile', 'potato', 'rice bag', 'roof tile', 'screw', 'shoe', 'spoon', 'spring roll', 'stair', 'stapler pin', 'straw', 'supermarket shelf', 'swan', 'tomato', 'watermelon', 'window', 'zebra'], val_label_list=['ant', 'bird', 'book', 'bottle cap', 'bullet', 'camel', 'chair', 'chicken wing', 'donut', 'donut holder', 'flamingo', 'flower', 'flower pot', 'grape', 'horse', 'kiwi', 'milk carton', 'oyster', 'oyster shell', 'package of fresh cut fruit', 'peach', 'pill', 'polka dot', 'prawn cracker', 'sausage', 'seagull', 'shallot', 'shirt', 'skateboard', 'toilet paper roll']) + +DEBUG  2024-03-29 17:46:26,832 | build model ... ... +DEBUG  2024-03-29 17:46:28,620 | build model, done. +INFO  2024-03-29 17:46:28,622 | number of params:236717952 +INFO  2024-03-29 17:46:28,625 | params before freezing: +{ + "transformer.level_embed": 1024, + "transformer.encoder.layers.0.self_attn.sampling_offsets.weight": 65536, + "transformer.encoder.layers.0.self_attn.sampling_offsets.bias": 256, + "transformer.encoder.layers.0.self_attn.attention_weights.weight": 32768, + "transformer.encoder.layers.0.self_attn.attention_weights.bias": 128, + "transformer.encoder.layers.0.self_attn.value_proj.weight": 65536, + "transformer.encoder.layers.0.self_attn.value_proj.bias": 256, + "transformer.encoder.layers.0.self_attn.output_proj.weight": 65536, + "transformer.encoder.layers.0.self_attn.output_proj.bias": 256, + "transformer.encoder.layers.0.norm1.weight": 256, + "transformer.encoder.layers.0.norm1.bias": 256, + "transformer.encoder.layers.0.linear1.weight": 524288, + "transformer.encoder.layers.0.linear1.bias": 2048, + "transformer.encoder.layers.0.linear2.weight": 524288, + "transformer.encoder.layers.0.linear2.bias": 256, + "transformer.encoder.layers.0.norm2.weight": 256, + "transformer.encoder.layers.0.norm2.bias": 256, + "transformer.encoder.layers.1.self_attn.sampling_offsets.weight": 65536, + "transformer.encoder.layers.1.self_attn.sampling_offsets.bias": 256, + "transformer.encoder.layers.1.self_attn.attention_weights.weight": 32768, + "transformer.encoder.layers.1.self_attn.attention_weights.bias": 128, + "transformer.encoder.layers.1.self_attn.value_proj.weight": 65536, + "transformer.encoder.layers.1.self_attn.value_proj.bias": 256, + "transformer.encoder.layers.1.self_attn.output_proj.weight": 65536, + "transformer.encoder.layers.1.self_attn.output_proj.bias": 256, + "transformer.encoder.layers.1.norm1.weight": 256, + "transformer.encoder.layers.1.norm1.bias": 256, + "transformer.encoder.layers.1.linear1.weight": 524288, + "transformer.encoder.layers.1.linear1.bias": 2048, + "transformer.encoder.layers.1.linear2.weight": 524288, + "transformer.encoder.layers.1.linear2.bias": 256, + "transformer.encoder.layers.1.norm2.weight": 256, + "transformer.encoder.layers.1.norm2.bias": 256, + "transformer.encoder.layers.2.self_attn.sampling_offsets.weight": 65536, + "transformer.encoder.layers.2.self_attn.sampling_offsets.bias": 256, + "transformer.encoder.layers.2.self_attn.attention_weights.weight": 32768, + "transformer.encoder.layers.2.self_attn.attention_weights.bias": 128, + "transformer.encoder.layers.2.self_attn.value_proj.weight": 65536, + "transformer.encoder.layers.2.self_attn.value_proj.bias": 256, + "transformer.encoder.layers.2.self_attn.output_proj.weight": 65536, + "transformer.encoder.layers.2.self_attn.output_proj.bias": 256, + "transformer.encoder.layers.2.norm1.weight": 256, + "transformer.encoder.layers.2.norm1.bias": 256, + "transformer.encoder.layers.2.linear1.weight": 524288, + "transformer.encoder.layers.2.linear1.bias": 2048, + "transformer.encoder.layers.2.linear2.weight": 524288, + "transformer.encoder.layers.2.linear2.bias": 256, + "transformer.encoder.layers.2.norm2.weight": 256, + "transformer.encoder.layers.2.norm2.bias": 256, + "transformer.encoder.layers.3.self_attn.sampling_offsets.weight": 65536, + "transformer.encoder.layers.3.self_attn.sampling_offsets.bias": 256, + "transformer.encoder.layers.3.self_attn.attention_weights.weight": 32768, + "transformer.encoder.layers.3.self_attn.attention_weights.bias": 128, + "transformer.encoder.layers.3.self_attn.value_proj.weight": 65536, + "transformer.encoder.layers.3.self_attn.value_proj.bias": 256, + "transformer.encoder.layers.3.self_attn.output_proj.weight": 65536, + "transformer.encoder.layers.3.self_attn.output_proj.bias": 256, + "transformer.encoder.layers.3.norm1.weight": 256, + "transformer.encoder.layers.3.norm1.bias": 256, + "transformer.encoder.layers.3.linear1.weight": 524288, + "transformer.encoder.layers.3.linear1.bias": 2048, + "transformer.encoder.layers.3.linear2.weight": 524288, + "transformer.encoder.layers.3.linear2.bias": 256, + "transformer.encoder.layers.3.norm2.weight": 256, + "transformer.encoder.layers.3.norm2.bias": 256, + "transformer.encoder.layers.4.self_attn.sampling_offsets.weight": 65536, + "transformer.encoder.layers.4.self_attn.sampling_offsets.bias": 256, + "transformer.encoder.layers.4.self_attn.attention_weights.weight": 32768, + "transformer.encoder.layers.4.self_attn.attention_weights.bias": 128, + "transformer.encoder.layers.4.self_attn.value_proj.weight": 65536, + "transformer.encoder.layers.4.self_attn.value_proj.bias": 256, + "transformer.encoder.layers.4.self_attn.output_proj.weight": 65536, + "transformer.encoder.layers.4.self_attn.output_proj.bias": 256, + "transformer.encoder.layers.4.norm1.weight": 256, + "transformer.encoder.layers.4.norm1.bias": 256, + "transformer.encoder.layers.4.linear1.weight": 524288, + "transformer.encoder.layers.4.linear1.bias": 2048, + "transformer.encoder.layers.4.linear2.weight": 524288, + "transformer.encoder.layers.4.linear2.bias": 256, + "transformer.encoder.layers.4.norm2.weight": 256, + "transformer.encoder.layers.4.norm2.bias": 256, + "transformer.encoder.layers.5.self_attn.sampling_offsets.weight": 65536, + "transformer.encoder.layers.5.self_attn.sampling_offsets.bias": 256, + "transformer.encoder.layers.5.self_attn.attention_weights.weight": 32768, + "transformer.encoder.layers.5.self_attn.attention_weights.bias": 128, + "transformer.encoder.layers.5.self_attn.value_proj.weight": 65536, + "transformer.encoder.layers.5.self_attn.value_proj.bias": 256, + "transformer.encoder.layers.5.self_attn.output_proj.weight": 65536, + "transformer.encoder.layers.5.self_attn.output_proj.bias": 256, + "transformer.encoder.layers.5.norm1.weight": 256, + "transformer.encoder.layers.5.norm1.bias": 256, + "transformer.encoder.layers.5.linear1.weight": 524288, + "transformer.encoder.layers.5.linear1.bias": 2048, + "transformer.encoder.layers.5.linear2.weight": 524288, + "transformer.encoder.layers.5.linear2.bias": 256, + "transformer.encoder.layers.5.norm2.weight": 256, + "transformer.encoder.layers.5.norm2.bias": 256, + "transformer.encoder.text_layers.0.self_attn.in_proj_weight": 196608, + "transformer.encoder.text_layers.0.self_attn.in_proj_bias": 768, + "transformer.encoder.text_layers.0.self_attn.out_proj.weight": 65536, + "transformer.encoder.text_layers.0.self_attn.out_proj.bias": 256, + "transformer.encoder.text_layers.0.linear1.weight": 262144, + "transformer.encoder.text_layers.0.linear1.bias": 1024, + "transformer.encoder.text_layers.0.linear2.weight": 262144, + "transformer.encoder.text_layers.0.linear2.bias": 256, + "transformer.encoder.text_layers.0.norm1.weight": 256, + "transformer.encoder.text_layers.0.norm1.bias": 256, + "transformer.encoder.text_layers.0.norm2.weight": 256, + "transformer.encoder.text_layers.0.norm2.bias": 256, + "transformer.encoder.text_layers.1.self_attn.in_proj_weight": 196608, + "transformer.encoder.text_layers.1.self_attn.in_proj_bias": 768, + "transformer.encoder.text_layers.1.self_attn.out_proj.weight": 65536, + "transformer.encoder.text_layers.1.self_attn.out_proj.bias": 256, + "transformer.encoder.text_layers.1.linear1.weight": 262144, + "transformer.encoder.text_layers.1.linear1.bias": 1024, + "transformer.encoder.text_layers.1.linear2.weight": 262144, + "transformer.encoder.text_layers.1.linear2.bias": 256, + "transformer.encoder.text_layers.1.norm1.weight": 256, + "transformer.encoder.text_layers.1.norm1.bias": 256, + "transformer.encoder.text_layers.1.norm2.weight": 256, + "transformer.encoder.text_layers.1.norm2.bias": 256, + "transformer.encoder.text_layers.2.self_attn.in_proj_weight": 196608, + "transformer.encoder.text_layers.2.self_attn.in_proj_bias": 768, + "transformer.encoder.text_layers.2.self_attn.out_proj.weight": 65536, + "transformer.encoder.text_layers.2.self_attn.out_proj.bias": 256, + "transformer.encoder.text_layers.2.linear1.weight": 262144, + "transformer.encoder.text_layers.2.linear1.bias": 1024, + "transformer.encoder.text_layers.2.linear2.weight": 262144, + "transformer.encoder.text_layers.2.linear2.bias": 256, + "transformer.encoder.text_layers.2.norm1.weight": 256, + "transformer.encoder.text_layers.2.norm1.bias": 256, + "transformer.encoder.text_layers.2.norm2.weight": 256, + "transformer.encoder.text_layers.2.norm2.bias": 256, + "transformer.encoder.text_layers.3.self_attn.in_proj_weight": 196608, + "transformer.encoder.text_layers.3.self_attn.in_proj_bias": 768, + "transformer.encoder.text_layers.3.self_attn.out_proj.weight": 65536, + "transformer.encoder.text_layers.3.self_attn.out_proj.bias": 256, + "transformer.encoder.text_layers.3.linear1.weight": 262144, + "transformer.encoder.text_layers.3.linear1.bias": 1024, + "transformer.encoder.text_layers.3.linear2.weight": 262144, + "transformer.encoder.text_layers.3.linear2.bias": 256, + "transformer.encoder.text_layers.3.norm1.weight": 256, + "transformer.encoder.text_layers.3.norm1.bias": 256, + "transformer.encoder.text_layers.3.norm2.weight": 256, + "transformer.encoder.text_layers.3.norm2.bias": 256, + "transformer.encoder.text_layers.4.self_attn.in_proj_weight": 196608, + "transformer.encoder.text_layers.4.self_attn.in_proj_bias": 768, + "transformer.encoder.text_layers.4.self_attn.out_proj.weight": 65536, + "transformer.encoder.text_layers.4.self_attn.out_proj.bias": 256, + "transformer.encoder.text_layers.4.linear1.weight": 262144, + "transformer.encoder.text_layers.4.linear1.bias": 1024, + "transformer.encoder.text_layers.4.linear2.weight": 262144, + "transformer.encoder.text_layers.4.linear2.bias": 256, + "transformer.encoder.text_layers.4.norm1.weight": 256, + "transformer.encoder.text_layers.4.norm1.bias": 256, + "transformer.encoder.text_layers.4.norm2.weight": 256, + "transformer.encoder.text_layers.4.norm2.bias": 256, + "transformer.encoder.text_layers.5.self_attn.in_proj_weight": 196608, + "transformer.encoder.text_layers.5.self_attn.in_proj_bias": 768, + "transformer.encoder.text_layers.5.self_attn.out_proj.weight": 65536, + "transformer.encoder.text_layers.5.self_attn.out_proj.bias": 256, + "transformer.encoder.text_layers.5.linear1.weight": 262144, + "transformer.encoder.text_layers.5.linear1.bias": 1024, + "transformer.encoder.text_layers.5.linear2.weight": 262144, + "transformer.encoder.text_layers.5.linear2.bias": 256, + "transformer.encoder.text_layers.5.norm1.weight": 256, + "transformer.encoder.text_layers.5.norm1.bias": 256, + "transformer.encoder.text_layers.5.norm2.weight": 256, + "transformer.encoder.text_layers.5.norm2.bias": 256, + "transformer.encoder.fusion_layers.0.gamma_v": 256, + "transformer.encoder.fusion_layers.0.gamma_l": 256, + "transformer.encoder.fusion_layers.0.layer_norm_v.weight": 256, + "transformer.encoder.fusion_layers.0.layer_norm_v.bias": 256, + "transformer.encoder.fusion_layers.0.layer_norm_l.weight": 256, + "transformer.encoder.fusion_layers.0.layer_norm_l.bias": 256, + "transformer.encoder.fusion_layers.0.attn.v_proj.weight": 262144, + "transformer.encoder.fusion_layers.0.attn.v_proj.bias": 1024, + "transformer.encoder.fusion_layers.0.attn.l_proj.weight": 262144, + "transformer.encoder.fusion_layers.0.attn.l_proj.bias": 1024, + "transformer.encoder.fusion_layers.0.attn.values_v_proj.weight": 262144, + "transformer.encoder.fusion_layers.0.attn.values_v_proj.bias": 1024, + "transformer.encoder.fusion_layers.0.attn.values_l_proj.weight": 262144, + "transformer.encoder.fusion_layers.0.attn.values_l_proj.bias": 1024, + "transformer.encoder.fusion_layers.0.attn.out_v_proj.weight": 262144, + "transformer.encoder.fusion_layers.0.attn.out_v_proj.bias": 256, + "transformer.encoder.fusion_layers.0.attn.out_l_proj.weight": 262144, + "transformer.encoder.fusion_layers.0.attn.out_l_proj.bias": 256, + "transformer.encoder.fusion_layers.1.gamma_v": 256, + "transformer.encoder.fusion_layers.1.gamma_l": 256, + "transformer.encoder.fusion_layers.1.layer_norm_v.weight": 256, + "transformer.encoder.fusion_layers.1.layer_norm_v.bias": 256, + "transformer.encoder.fusion_layers.1.layer_norm_l.weight": 256, + "transformer.encoder.fusion_layers.1.layer_norm_l.bias": 256, + "transformer.encoder.fusion_layers.1.attn.v_proj.weight": 262144, + "transformer.encoder.fusion_layers.1.attn.v_proj.bias": 1024, + "transformer.encoder.fusion_layers.1.attn.l_proj.weight": 262144, + "transformer.encoder.fusion_layers.1.attn.l_proj.bias": 1024, + "transformer.encoder.fusion_layers.1.attn.values_v_proj.weight": 262144, + "transformer.encoder.fusion_layers.1.attn.values_v_proj.bias": 1024, + "transformer.encoder.fusion_layers.1.attn.values_l_proj.weight": 262144, + "transformer.encoder.fusion_layers.1.attn.values_l_proj.bias": 1024, + "transformer.encoder.fusion_layers.1.attn.out_v_proj.weight": 262144, + "transformer.encoder.fusion_layers.1.attn.out_v_proj.bias": 256, + "transformer.encoder.fusion_layers.1.attn.out_l_proj.weight": 262144, + "transformer.encoder.fusion_layers.1.attn.out_l_proj.bias": 256, + "transformer.encoder.fusion_layers.2.gamma_v": 256, + "transformer.encoder.fusion_layers.2.gamma_l": 256, + "transformer.encoder.fusion_layers.2.layer_norm_v.weight": 256, + "transformer.encoder.fusion_layers.2.layer_norm_v.bias": 256, + "transformer.encoder.fusion_layers.2.layer_norm_l.weight": 256, + "transformer.encoder.fusion_layers.2.layer_norm_l.bias": 256, + "transformer.encoder.fusion_layers.2.attn.v_proj.weight": 262144, + "transformer.encoder.fusion_layers.2.attn.v_proj.bias": 1024, + "transformer.encoder.fusion_layers.2.attn.l_proj.weight": 262144, + "transformer.encoder.fusion_layers.2.attn.l_proj.bias": 1024, + "transformer.encoder.fusion_layers.2.attn.values_v_proj.weight": 262144, + "transformer.encoder.fusion_layers.2.attn.values_v_proj.bias": 1024, + "transformer.encoder.fusion_layers.2.attn.values_l_proj.weight": 262144, + "transformer.encoder.fusion_layers.2.attn.values_l_proj.bias": 1024, + "transformer.encoder.fusion_layers.2.attn.out_v_proj.weight": 262144, + "transformer.encoder.fusion_layers.2.attn.out_v_proj.bias": 256, + "transformer.encoder.fusion_layers.2.attn.out_l_proj.weight": 262144, + "transformer.encoder.fusion_layers.2.attn.out_l_proj.bias": 256, + "transformer.encoder.fusion_layers.3.gamma_v": 256, + "transformer.encoder.fusion_layers.3.gamma_l": 256, + "transformer.encoder.fusion_layers.3.layer_norm_v.weight": 256, + "transformer.encoder.fusion_layers.3.layer_norm_v.bias": 256, + "transformer.encoder.fusion_layers.3.layer_norm_l.weight": 256, + "transformer.encoder.fusion_layers.3.layer_norm_l.bias": 256, + "transformer.encoder.fusion_layers.3.attn.v_proj.weight": 262144, + "transformer.encoder.fusion_layers.3.attn.v_proj.bias": 1024, + "transformer.encoder.fusion_layers.3.attn.l_proj.weight": 262144, + "transformer.encoder.fusion_layers.3.attn.l_proj.bias": 1024, + "transformer.encoder.fusion_layers.3.attn.values_v_proj.weight": 262144, + "transformer.encoder.fusion_layers.3.attn.values_v_proj.bias": 1024, + "transformer.encoder.fusion_layers.3.attn.values_l_proj.weight": 262144, + "transformer.encoder.fusion_layers.3.attn.values_l_proj.bias": 1024, + "transformer.encoder.fusion_layers.3.attn.out_v_proj.weight": 262144, + "transformer.encoder.fusion_layers.3.attn.out_v_proj.bias": 256, + "transformer.encoder.fusion_layers.3.attn.out_l_proj.weight": 262144, + "transformer.encoder.fusion_layers.3.attn.out_l_proj.bias": 256, + "transformer.encoder.fusion_layers.4.gamma_v": 256, + "transformer.encoder.fusion_layers.4.gamma_l": 256, + "transformer.encoder.fusion_layers.4.layer_norm_v.weight": 256, + "transformer.encoder.fusion_layers.4.layer_norm_v.bias": 256, + "transformer.encoder.fusion_layers.4.layer_norm_l.weight": 256, + "transformer.encoder.fusion_layers.4.layer_norm_l.bias": 256, + "transformer.encoder.fusion_layers.4.attn.v_proj.weight": 262144, + "transformer.encoder.fusion_layers.4.attn.v_proj.bias": 1024, + "transformer.encoder.fusion_layers.4.attn.l_proj.weight": 262144, + "transformer.encoder.fusion_layers.4.attn.l_proj.bias": 1024, + "transformer.encoder.fusion_layers.4.attn.values_v_proj.weight": 262144, + "transformer.encoder.fusion_layers.4.attn.values_v_proj.bias": 1024, + "transformer.encoder.fusion_layers.4.attn.values_l_proj.weight": 262144, + "transformer.encoder.fusion_layers.4.attn.values_l_proj.bias": 1024, + "transformer.encoder.fusion_layers.4.attn.out_v_proj.weight": 262144, + "transformer.encoder.fusion_layers.4.attn.out_v_proj.bias": 256, + "transformer.encoder.fusion_layers.4.attn.out_l_proj.weight": 262144, + "transformer.encoder.fusion_layers.4.attn.out_l_proj.bias": 256, + "transformer.encoder.fusion_layers.5.gamma_v": 256, + "transformer.encoder.fusion_layers.5.gamma_l": 256, + "transformer.encoder.fusion_layers.5.layer_norm_v.weight": 256, + "transformer.encoder.fusion_layers.5.layer_norm_v.bias": 256, + "transformer.encoder.fusion_layers.5.layer_norm_l.weight": 256, + "transformer.encoder.fusion_layers.5.layer_norm_l.bias": 256, + "transformer.encoder.fusion_layers.5.attn.v_proj.weight": 262144, + "transformer.encoder.fusion_layers.5.attn.v_proj.bias": 1024, + "transformer.encoder.fusion_layers.5.attn.l_proj.weight": 262144, + "transformer.encoder.fusion_layers.5.attn.l_proj.bias": 1024, + "transformer.encoder.fusion_layers.5.attn.values_v_proj.weight": 262144, + "transformer.encoder.fusion_layers.5.attn.values_v_proj.bias": 1024, + "transformer.encoder.fusion_layers.5.attn.values_l_proj.weight": 262144, + "transformer.encoder.fusion_layers.5.attn.values_l_proj.bias": 1024, + "transformer.encoder.fusion_layers.5.attn.out_v_proj.weight": 262144, + "transformer.encoder.fusion_layers.5.attn.out_v_proj.bias": 256, + "transformer.encoder.fusion_layers.5.attn.out_l_proj.weight": 262144, + "transformer.encoder.fusion_layers.5.attn.out_l_proj.bias": 256, + "transformer.decoder.layers.0.cross_attn.sampling_offsets.weight": 65536, + "transformer.decoder.layers.0.cross_attn.sampling_offsets.bias": 256, + "transformer.decoder.layers.0.cross_attn.attention_weights.weight": 32768, + "transformer.decoder.layers.0.cross_attn.attention_weights.bias": 128, + "transformer.decoder.layers.0.cross_attn.value_proj.weight": 65536, + "transformer.decoder.layers.0.cross_attn.value_proj.bias": 256, + "transformer.decoder.layers.0.cross_attn.output_proj.weight": 65536, + "transformer.decoder.layers.0.cross_attn.output_proj.bias": 256, + "transformer.decoder.layers.0.norm1.weight": 256, + "transformer.decoder.layers.0.norm1.bias": 256, + "transformer.decoder.layers.0.ca_text.in_proj_weight": 196608, + "transformer.decoder.layers.0.ca_text.in_proj_bias": 768, + "transformer.decoder.layers.0.ca_text.out_proj.weight": 65536, + "transformer.decoder.layers.0.ca_text.out_proj.bias": 256, + "transformer.decoder.layers.0.catext_norm.weight": 256, + "transformer.decoder.layers.0.catext_norm.bias": 256, + "transformer.decoder.layers.0.self_attn.in_proj_weight": 196608, + "transformer.decoder.layers.0.self_attn.in_proj_bias": 768, + "transformer.decoder.layers.0.self_attn.out_proj.weight": 65536, + "transformer.decoder.layers.0.self_attn.out_proj.bias": 256, + "transformer.decoder.layers.0.norm2.weight": 256, + "transformer.decoder.layers.0.norm2.bias": 256, + "transformer.decoder.layers.0.linear1.weight": 524288, + "transformer.decoder.layers.0.linear1.bias": 2048, + "transformer.decoder.layers.0.linear2.weight": 524288, + "transformer.decoder.layers.0.linear2.bias": 256, + "transformer.decoder.layers.0.norm3.weight": 256, + "transformer.decoder.layers.0.norm3.bias": 256, + "transformer.decoder.layers.1.cross_attn.sampling_offsets.weight": 65536, + "transformer.decoder.layers.1.cross_attn.sampling_offsets.bias": 256, + "transformer.decoder.layers.1.cross_attn.attention_weights.weight": 32768, + "transformer.decoder.layers.1.cross_attn.attention_weights.bias": 128, + "transformer.decoder.layers.1.cross_attn.value_proj.weight": 65536, + "transformer.decoder.layers.1.cross_attn.value_proj.bias": 256, + "transformer.decoder.layers.1.cross_attn.output_proj.weight": 65536, + "transformer.decoder.layers.1.cross_attn.output_proj.bias": 256, + "transformer.decoder.layers.1.norm1.weight": 256, + "transformer.decoder.layers.1.norm1.bias": 256, + "transformer.decoder.layers.1.ca_text.in_proj_weight": 196608, + "transformer.decoder.layers.1.ca_text.in_proj_bias": 768, + "transformer.decoder.layers.1.ca_text.out_proj.weight": 65536, + "transformer.decoder.layers.1.ca_text.out_proj.bias": 256, + "transformer.decoder.layers.1.catext_norm.weight": 256, + "transformer.decoder.layers.1.catext_norm.bias": 256, + "transformer.decoder.layers.1.self_attn.in_proj_weight": 196608, + "transformer.decoder.layers.1.self_attn.in_proj_bias": 768, + "transformer.decoder.layers.1.self_attn.out_proj.weight": 65536, + "transformer.decoder.layers.1.self_attn.out_proj.bias": 256, + "transformer.decoder.layers.1.norm2.weight": 256, + "transformer.decoder.layers.1.norm2.bias": 256, + "transformer.decoder.layers.1.linear1.weight": 524288, + "transformer.decoder.layers.1.linear1.bias": 2048, + "transformer.decoder.layers.1.linear2.weight": 524288, + "transformer.decoder.layers.1.linear2.bias": 256, + "transformer.decoder.layers.1.norm3.weight": 256, + "transformer.decoder.layers.1.norm3.bias": 256, + "transformer.decoder.layers.2.cross_attn.sampling_offsets.weight": 65536, + "transformer.decoder.layers.2.cross_attn.sampling_offsets.bias": 256, + "transformer.decoder.layers.2.cross_attn.attention_weights.weight": 32768, + "transformer.decoder.layers.2.cross_attn.attention_weights.bias": 128, + "transformer.decoder.layers.2.cross_attn.value_proj.weight": 65536, + "transformer.decoder.layers.2.cross_attn.value_proj.bias": 256, + "transformer.decoder.layers.2.cross_attn.output_proj.weight": 65536, + "transformer.decoder.layers.2.cross_attn.output_proj.bias": 256, + "transformer.decoder.layers.2.norm1.weight": 256, + "transformer.decoder.layers.2.norm1.bias": 256, + "transformer.decoder.layers.2.ca_text.in_proj_weight": 196608, + "transformer.decoder.layers.2.ca_text.in_proj_bias": 768, + "transformer.decoder.layers.2.ca_text.out_proj.weight": 65536, + "transformer.decoder.layers.2.ca_text.out_proj.bias": 256, + "transformer.decoder.layers.2.catext_norm.weight": 256, + "transformer.decoder.layers.2.catext_norm.bias": 256, + "transformer.decoder.layers.2.self_attn.in_proj_weight": 196608, + "transformer.decoder.layers.2.self_attn.in_proj_bias": 768, + "transformer.decoder.layers.2.self_attn.out_proj.weight": 65536, + "transformer.decoder.layers.2.self_attn.out_proj.bias": 256, + "transformer.decoder.layers.2.norm2.weight": 256, + "transformer.decoder.layers.2.norm2.bias": 256, + "transformer.decoder.layers.2.linear1.weight": 524288, + "transformer.decoder.layers.2.linear1.bias": 2048, + "transformer.decoder.layers.2.linear2.weight": 524288, + "transformer.decoder.layers.2.linear2.bias": 256, + "transformer.decoder.layers.2.norm3.weight": 256, + "transformer.decoder.layers.2.norm3.bias": 256, + "transformer.decoder.layers.3.cross_attn.sampling_offsets.weight": 65536, + "transformer.decoder.layers.3.cross_attn.sampling_offsets.bias": 256, + "transformer.decoder.layers.3.cross_attn.attention_weights.weight": 32768, + "transformer.decoder.layers.3.cross_attn.attention_weights.bias": 128, + "transformer.decoder.layers.3.cross_attn.value_proj.weight": 65536, + "transformer.decoder.layers.3.cross_attn.value_proj.bias": 256, + "transformer.decoder.layers.3.cross_attn.output_proj.weight": 65536, + "transformer.decoder.layers.3.cross_attn.output_proj.bias": 256, + "transformer.decoder.layers.3.norm1.weight": 256, + "transformer.decoder.layers.3.norm1.bias": 256, + "transformer.decoder.layers.3.ca_text.in_proj_weight": 196608, + "transformer.decoder.layers.3.ca_text.in_proj_bias": 768, + "transformer.decoder.layers.3.ca_text.out_proj.weight": 65536, + "transformer.decoder.layers.3.ca_text.out_proj.bias": 256, + "transformer.decoder.layers.3.catext_norm.weight": 256, + "transformer.decoder.layers.3.catext_norm.bias": 256, + "transformer.decoder.layers.3.self_attn.in_proj_weight": 196608, + "transformer.decoder.layers.3.self_attn.in_proj_bias": 768, + "transformer.decoder.layers.3.self_attn.out_proj.weight": 65536, + "transformer.decoder.layers.3.self_attn.out_proj.bias": 256, + "transformer.decoder.layers.3.norm2.weight": 256, + "transformer.decoder.layers.3.norm2.bias": 256, + "transformer.decoder.layers.3.linear1.weight": 524288, + "transformer.decoder.layers.3.linear1.bias": 2048, + "transformer.decoder.layers.3.linear2.weight": 524288, + "transformer.decoder.layers.3.linear2.bias": 256, + "transformer.decoder.layers.3.norm3.weight": 256, + "transformer.decoder.layers.3.norm3.bias": 256, + "transformer.decoder.layers.4.cross_attn.sampling_offsets.weight": 65536, + "transformer.decoder.layers.4.cross_attn.sampling_offsets.bias": 256, + "transformer.decoder.layers.4.cross_attn.attention_weights.weight": 32768, + "transformer.decoder.layers.4.cross_attn.attention_weights.bias": 128, + "transformer.decoder.layers.4.cross_attn.value_proj.weight": 65536, + "transformer.decoder.layers.4.cross_attn.value_proj.bias": 256, + "transformer.decoder.layers.4.cross_attn.output_proj.weight": 65536, + "transformer.decoder.layers.4.cross_attn.output_proj.bias": 256, + "transformer.decoder.layers.4.norm1.weight": 256, + "transformer.decoder.layers.4.norm1.bias": 256, + "transformer.decoder.layers.4.ca_text.in_proj_weight": 196608, + "transformer.decoder.layers.4.ca_text.in_proj_bias": 768, + "transformer.decoder.layers.4.ca_text.out_proj.weight": 65536, + "transformer.decoder.layers.4.ca_text.out_proj.bias": 256, + "transformer.decoder.layers.4.catext_norm.weight": 256, + "transformer.decoder.layers.4.catext_norm.bias": 256, + "transformer.decoder.layers.4.self_attn.in_proj_weight": 196608, + "transformer.decoder.layers.4.self_attn.in_proj_bias": 768, + "transformer.decoder.layers.4.self_attn.out_proj.weight": 65536, + "transformer.decoder.layers.4.self_attn.out_proj.bias": 256, + "transformer.decoder.layers.4.norm2.weight": 256, + "transformer.decoder.layers.4.norm2.bias": 256, + "transformer.decoder.layers.4.linear1.weight": 524288, + "transformer.decoder.layers.4.linear1.bias": 2048, + "transformer.decoder.layers.4.linear2.weight": 524288, + "transformer.decoder.layers.4.linear2.bias": 256, + "transformer.decoder.layers.4.norm3.weight": 256, + "transformer.decoder.layers.4.norm3.bias": 256, + "transformer.decoder.layers.5.cross_attn.sampling_offsets.weight": 65536, + "transformer.decoder.layers.5.cross_attn.sampling_offsets.bias": 256, + "transformer.decoder.layers.5.cross_attn.attention_weights.weight": 32768, + "transformer.decoder.layers.5.cross_attn.attention_weights.bias": 128, + "transformer.decoder.layers.5.cross_attn.value_proj.weight": 65536, + "transformer.decoder.layers.5.cross_attn.value_proj.bias": 256, + "transformer.decoder.layers.5.cross_attn.output_proj.weight": 65536, + "transformer.decoder.layers.5.cross_attn.output_proj.bias": 256, + "transformer.decoder.layers.5.norm1.weight": 256, + "transformer.decoder.layers.5.norm1.bias": 256, + "transformer.decoder.layers.5.ca_text.in_proj_weight": 196608, + "transformer.decoder.layers.5.ca_text.in_proj_bias": 768, + "transformer.decoder.layers.5.ca_text.out_proj.weight": 65536, + "transformer.decoder.layers.5.ca_text.out_proj.bias": 256, + "transformer.decoder.layers.5.catext_norm.weight": 256, + "transformer.decoder.layers.5.catext_norm.bias": 256, + "transformer.decoder.layers.5.self_attn.in_proj_weight": 196608, + "transformer.decoder.layers.5.self_attn.in_proj_bias": 768, + "transformer.decoder.layers.5.self_attn.out_proj.weight": 65536, + "transformer.decoder.layers.5.self_attn.out_proj.bias": 256, + "transformer.decoder.layers.5.norm2.weight": 256, + "transformer.decoder.layers.5.norm2.bias": 256, + "transformer.decoder.layers.5.linear1.weight": 524288, + "transformer.decoder.layers.5.linear1.bias": 2048, + "transformer.decoder.layers.5.linear2.weight": 524288, + "transformer.decoder.layers.5.linear2.bias": 256, + "transformer.decoder.layers.5.norm3.weight": 256, + "transformer.decoder.layers.5.norm3.bias": 256, + "transformer.decoder.norm.weight": 256, + "transformer.decoder.norm.bias": 256, + "transformer.decoder.ref_point_head.layers.0.weight": 131072, + "transformer.decoder.ref_point_head.layers.0.bias": 256, + "transformer.decoder.ref_point_head.layers.1.weight": 65536, + "transformer.decoder.ref_point_head.layers.1.bias": 256, + "transformer.decoder.bbox_embed.0.layers.0.weight": 65536, + "transformer.decoder.bbox_embed.0.layers.0.bias": 256, + "transformer.decoder.bbox_embed.0.layers.1.weight": 65536, + "transformer.decoder.bbox_embed.0.layers.1.bias": 256, + "transformer.decoder.bbox_embed.0.layers.2.weight": 1024, + "transformer.decoder.bbox_embed.0.layers.2.bias": 4, + "transformer.tgt_embed.weight": 230400, + "transformer.enc_output.weight": 65536, + "transformer.enc_output.bias": 256, + "transformer.enc_output_norm.weight": 256, + "transformer.enc_output_norm.bias": 256, + "transformer.enc_out_bbox_embed.layers.0.weight": 65536, + "transformer.enc_out_bbox_embed.layers.0.bias": 256, + "transformer.enc_out_bbox_embed.layers.1.weight": 65536, + "transformer.enc_out_bbox_embed.layers.1.bias": 256, + "transformer.enc_out_bbox_embed.layers.2.weight": 1024, + "transformer.enc_out_bbox_embed.layers.2.bias": 4, + "feature_map_proj.weight": 458752, + "feature_map_proj.bias": 256, + "feature_map_encoder.layers.0.norm1.weight": 256, + "feature_map_encoder.layers.0.norm1.bias": 256, + "feature_map_encoder.layers.0.norm2.weight": 256, + "feature_map_encoder.layers.0.norm2.bias": 256, + "feature_map_encoder.layers.0.self_attn.in_proj_weight": 196608, + "feature_map_encoder.layers.0.self_attn.in_proj_bias": 768, + "feature_map_encoder.layers.0.self_attn.out_proj.weight": 65536, + "feature_map_encoder.layers.0.self_attn.out_proj.bias": 256, + "feature_map_encoder.layers.0.mlp.linear1.weight": 524288, + "feature_map_encoder.layers.0.mlp.linear1.bias": 2048, + "feature_map_encoder.layers.0.mlp.linear2.weight": 524288, + "feature_map_encoder.layers.0.mlp.linear2.bias": 256, + "feature_map_encoder.layers.1.norm1.weight": 256, + "feature_map_encoder.layers.1.norm1.bias": 256, + "feature_map_encoder.layers.1.norm2.weight": 256, + "feature_map_encoder.layers.1.norm2.bias": 256, + "feature_map_encoder.layers.1.self_attn.in_proj_weight": 196608, + "feature_map_encoder.layers.1.self_attn.in_proj_bias": 768, + "feature_map_encoder.layers.1.self_attn.out_proj.weight": 65536, + "feature_map_encoder.layers.1.self_attn.out_proj.bias": 256, + "feature_map_encoder.layers.1.mlp.linear1.weight": 524288, + "feature_map_encoder.layers.1.mlp.linear1.bias": 2048, + "feature_map_encoder.layers.1.mlp.linear2.weight": 524288, + "feature_map_encoder.layers.1.mlp.linear2.bias": 256, + "feature_map_encoder.layers.2.norm1.weight": 256, + "feature_map_encoder.layers.2.norm1.bias": 256, + "feature_map_encoder.layers.2.norm2.weight": 256, + "feature_map_encoder.layers.2.norm2.bias": 256, + "feature_map_encoder.layers.2.self_attn.in_proj_weight": 196608, + "feature_map_encoder.layers.2.self_attn.in_proj_bias": 768, + "feature_map_encoder.layers.2.self_attn.out_proj.weight": 65536, + "feature_map_encoder.layers.2.self_attn.out_proj.bias": 256, + "feature_map_encoder.layers.2.mlp.linear1.weight": 524288, + "feature_map_encoder.layers.2.mlp.linear1.bias": 2048, + "feature_map_encoder.layers.2.mlp.linear2.weight": 524288, + "feature_map_encoder.layers.2.mlp.linear2.bias": 256, + "feature_map_encoder.norm.weight": 256, + "feature_map_encoder.norm.bias": 256, + "bert.embeddings.word_embeddings.weight": 23440896, + "bert.embeddings.position_embeddings.weight": 393216, + "bert.embeddings.token_type_embeddings.weight": 1536, + "bert.embeddings.LayerNorm.weight": 768, + "bert.embeddings.LayerNorm.bias": 768, + "bert.encoder.layer.0.attention.self.query.weight": 589824, + "bert.encoder.layer.0.attention.self.query.bias": 768, + "bert.encoder.layer.0.attention.self.key.weight": 589824, + "bert.encoder.layer.0.attention.self.key.bias": 768, + "bert.encoder.layer.0.attention.self.value.weight": 589824, + "bert.encoder.layer.0.attention.self.value.bias": 768, + "bert.encoder.layer.0.attention.output.dense.weight": 589824, + "bert.encoder.layer.0.attention.output.dense.bias": 768, + "bert.encoder.layer.0.attention.output.LayerNorm.weight": 768, + "bert.encoder.layer.0.attention.output.LayerNorm.bias": 768, + "bert.encoder.layer.0.intermediate.dense.weight": 2359296, + "bert.encoder.layer.0.intermediate.dense.bias": 3072, + "bert.encoder.layer.0.output.dense.weight": 2359296, + "bert.encoder.layer.0.output.dense.bias": 768, + "bert.encoder.layer.0.output.LayerNorm.weight": 768, + "bert.encoder.layer.0.output.LayerNorm.bias": 768, + "bert.encoder.layer.1.attention.self.query.weight": 589824, + "bert.encoder.layer.1.attention.self.query.bias": 768, + "bert.encoder.layer.1.attention.self.key.weight": 589824, + "bert.encoder.layer.1.attention.self.key.bias": 768, + "bert.encoder.layer.1.attention.self.value.weight": 589824, + "bert.encoder.layer.1.attention.self.value.bias": 768, + "bert.encoder.layer.1.attention.output.dense.weight": 589824, + "bert.encoder.layer.1.attention.output.dense.bias": 768, + "bert.encoder.layer.1.attention.output.LayerNorm.weight": 768, + "bert.encoder.layer.1.attention.output.LayerNorm.bias": 768, + "bert.encoder.layer.1.intermediate.dense.weight": 2359296, + "bert.encoder.layer.1.intermediate.dense.bias": 3072, + "bert.encoder.layer.1.output.dense.weight": 2359296, + "bert.encoder.layer.1.output.dense.bias": 768, + "bert.encoder.layer.1.output.LayerNorm.weight": 768, + "bert.encoder.layer.1.output.LayerNorm.bias": 768, + "bert.encoder.layer.2.attention.self.query.weight": 589824, + "bert.encoder.layer.2.attention.self.query.bias": 768, + "bert.encoder.layer.2.attention.self.key.weight": 589824, + "bert.encoder.layer.2.attention.self.key.bias": 768, + "bert.encoder.layer.2.attention.self.value.weight": 589824, + "bert.encoder.layer.2.attention.self.value.bias": 768, + "bert.encoder.layer.2.attention.output.dense.weight": 589824, + "bert.encoder.layer.2.attention.output.dense.bias": 768, + "bert.encoder.layer.2.attention.output.LayerNorm.weight": 768, + "bert.encoder.layer.2.attention.output.LayerNorm.bias": 768, + "bert.encoder.layer.2.intermediate.dense.weight": 2359296, + "bert.encoder.layer.2.intermediate.dense.bias": 3072, + "bert.encoder.layer.2.output.dense.weight": 2359296, + "bert.encoder.layer.2.output.dense.bias": 768, + "bert.encoder.layer.2.output.LayerNorm.weight": 768, + "bert.encoder.layer.2.output.LayerNorm.bias": 768, + "bert.encoder.layer.3.attention.self.query.weight": 589824, + "bert.encoder.layer.3.attention.self.query.bias": 768, + "bert.encoder.layer.3.attention.self.key.weight": 589824, + "bert.encoder.layer.3.attention.self.key.bias": 768, + "bert.encoder.layer.3.attention.self.value.weight": 589824, + "bert.encoder.layer.3.attention.self.value.bias": 768, + "bert.encoder.layer.3.attention.output.dense.weight": 589824, + "bert.encoder.layer.3.attention.output.dense.bias": 768, + "bert.encoder.layer.3.attention.output.LayerNorm.weight": 768, + "bert.encoder.layer.3.attention.output.LayerNorm.bias": 768, + "bert.encoder.layer.3.intermediate.dense.weight": 2359296, + "bert.encoder.layer.3.intermediate.dense.bias": 3072, + "bert.encoder.layer.3.output.dense.weight": 2359296, + "bert.encoder.layer.3.output.dense.bias": 768, + "bert.encoder.layer.3.output.LayerNorm.weight": 768, + "bert.encoder.layer.3.output.LayerNorm.bias": 768, + "bert.encoder.layer.4.attention.self.query.weight": 589824, + "bert.encoder.layer.4.attention.self.query.bias": 768, + "bert.encoder.layer.4.attention.self.key.weight": 589824, + "bert.encoder.layer.4.attention.self.key.bias": 768, + "bert.encoder.layer.4.attention.self.value.weight": 589824, + "bert.encoder.layer.4.attention.self.value.bias": 768, + "bert.encoder.layer.4.attention.output.dense.weight": 589824, + "bert.encoder.layer.4.attention.output.dense.bias": 768, + "bert.encoder.layer.4.attention.output.LayerNorm.weight": 768, + "bert.encoder.layer.4.attention.output.LayerNorm.bias": 768, + "bert.encoder.layer.4.intermediate.dense.weight": 2359296, + "bert.encoder.layer.4.intermediate.dense.bias": 3072, + "bert.encoder.layer.4.output.dense.weight": 2359296, + "bert.encoder.layer.4.output.dense.bias": 768, + "bert.encoder.layer.4.output.LayerNorm.weight": 768, + "bert.encoder.layer.4.output.LayerNorm.bias": 768, + "bert.encoder.layer.5.attention.self.query.weight": 589824, + "bert.encoder.layer.5.attention.self.query.bias": 768, + "bert.encoder.layer.5.attention.self.key.weight": 589824, + "bert.encoder.layer.5.attention.self.key.bias": 768, + "bert.encoder.layer.5.attention.self.value.weight": 589824, + "bert.encoder.layer.5.attention.self.value.bias": 768, + "bert.encoder.layer.5.attention.output.dense.weight": 589824, + "bert.encoder.layer.5.attention.output.dense.bias": 768, + "bert.encoder.layer.5.attention.output.LayerNorm.weight": 768, + "bert.encoder.layer.5.attention.output.LayerNorm.bias": 768, + "bert.encoder.layer.5.intermediate.dense.weight": 2359296, + "bert.encoder.layer.5.intermediate.dense.bias": 3072, + "bert.encoder.layer.5.output.dense.weight": 2359296, + "bert.encoder.layer.5.output.dense.bias": 768, + "bert.encoder.layer.5.output.LayerNorm.weight": 768, + "bert.encoder.layer.5.output.LayerNorm.bias": 768, + "bert.encoder.layer.6.attention.self.query.weight": 589824, + "bert.encoder.layer.6.attention.self.query.bias": 768, + "bert.encoder.layer.6.attention.self.key.weight": 589824, + "bert.encoder.layer.6.attention.self.key.bias": 768, + "bert.encoder.layer.6.attention.self.value.weight": 589824, + "bert.encoder.layer.6.attention.self.value.bias": 768, + "bert.encoder.layer.6.attention.output.dense.weight": 589824, + "bert.encoder.layer.6.attention.output.dense.bias": 768, + "bert.encoder.layer.6.attention.output.LayerNorm.weight": 768, + "bert.encoder.layer.6.attention.output.LayerNorm.bias": 768, + "bert.encoder.layer.6.intermediate.dense.weight": 2359296, + "bert.encoder.layer.6.intermediate.dense.bias": 3072, + "bert.encoder.layer.6.output.dense.weight": 2359296, + "bert.encoder.layer.6.output.dense.bias": 768, + "bert.encoder.layer.6.output.LayerNorm.weight": 768, + "bert.encoder.layer.6.output.LayerNorm.bias": 768, + "bert.encoder.layer.7.attention.self.query.weight": 589824, + "bert.encoder.layer.7.attention.self.query.bias": 768, + "bert.encoder.layer.7.attention.self.key.weight": 589824, + "bert.encoder.layer.7.attention.self.key.bias": 768, + "bert.encoder.layer.7.attention.self.value.weight": 589824, + "bert.encoder.layer.7.attention.self.value.bias": 768, + "bert.encoder.layer.7.attention.output.dense.weight": 589824, + "bert.encoder.layer.7.attention.output.dense.bias": 768, + "bert.encoder.layer.7.attention.output.LayerNorm.weight": 768, + "bert.encoder.layer.7.attention.output.LayerNorm.bias": 768, + "bert.encoder.layer.7.intermediate.dense.weight": 2359296, + "bert.encoder.layer.7.intermediate.dense.bias": 3072, + "bert.encoder.layer.7.output.dense.weight": 2359296, + "bert.encoder.layer.7.output.dense.bias": 768, + "bert.encoder.layer.7.output.LayerNorm.weight": 768, + "bert.encoder.layer.7.output.LayerNorm.bias": 768, + "bert.encoder.layer.8.attention.self.query.weight": 589824, + "bert.encoder.layer.8.attention.self.query.bias": 768, + "bert.encoder.layer.8.attention.self.key.weight": 589824, + "bert.encoder.layer.8.attention.self.key.bias": 768, + "bert.encoder.layer.8.attention.self.value.weight": 589824, + "bert.encoder.layer.8.attention.self.value.bias": 768, + "bert.encoder.layer.8.attention.output.dense.weight": 589824, + "bert.encoder.layer.8.attention.output.dense.bias": 768, + "bert.encoder.layer.8.attention.output.LayerNorm.weight": 768, + "bert.encoder.layer.8.attention.output.LayerNorm.bias": 768, + "bert.encoder.layer.8.intermediate.dense.weight": 2359296, + "bert.encoder.layer.8.intermediate.dense.bias": 3072, + "bert.encoder.layer.8.output.dense.weight": 2359296, + "bert.encoder.layer.8.output.dense.bias": 768, + "bert.encoder.layer.8.output.LayerNorm.weight": 768, + "bert.encoder.layer.8.output.LayerNorm.bias": 768, + "bert.encoder.layer.9.attention.self.query.weight": 589824, + "bert.encoder.layer.9.attention.self.query.bias": 768, + "bert.encoder.layer.9.attention.self.key.weight": 589824, + "bert.encoder.layer.9.attention.self.key.bias": 768, + "bert.encoder.layer.9.attention.self.value.weight": 589824, + "bert.encoder.layer.9.attention.self.value.bias": 768, + "bert.encoder.layer.9.attention.output.dense.weight": 589824, + "bert.encoder.layer.9.attention.output.dense.bias": 768, + "bert.encoder.layer.9.attention.output.LayerNorm.weight": 768, + "bert.encoder.layer.9.attention.output.LayerNorm.bias": 768, + "bert.encoder.layer.9.intermediate.dense.weight": 2359296, + "bert.encoder.layer.9.intermediate.dense.bias": 3072, + "bert.encoder.layer.9.output.dense.weight": 2359296, + "bert.encoder.layer.9.output.dense.bias": 768, + "bert.encoder.layer.9.output.LayerNorm.weight": 768, + "bert.encoder.layer.9.output.LayerNorm.bias": 768, + "bert.encoder.layer.10.attention.self.query.weight": 589824, + "bert.encoder.layer.10.attention.self.query.bias": 768, + "bert.encoder.layer.10.attention.self.key.weight": 589824, + "bert.encoder.layer.10.attention.self.key.bias": 768, + "bert.encoder.layer.10.attention.self.value.weight": 589824, + "bert.encoder.layer.10.attention.self.value.bias": 768, + "bert.encoder.layer.10.attention.output.dense.weight": 589824, + "bert.encoder.layer.10.attention.output.dense.bias": 768, + "bert.encoder.layer.10.attention.output.LayerNorm.weight": 768, + "bert.encoder.layer.10.attention.output.LayerNorm.bias": 768, + "bert.encoder.layer.10.intermediate.dense.weight": 2359296, + "bert.encoder.layer.10.intermediate.dense.bias": 3072, + "bert.encoder.layer.10.output.dense.weight": 2359296, + "bert.encoder.layer.10.output.dense.bias": 768, + "bert.encoder.layer.10.output.LayerNorm.weight": 768, + "bert.encoder.layer.10.output.LayerNorm.bias": 768, + "bert.encoder.layer.11.attention.self.query.weight": 589824, + "bert.encoder.layer.11.attention.self.query.bias": 768, + "bert.encoder.layer.11.attention.self.key.weight": 589824, + "bert.encoder.layer.11.attention.self.key.bias": 768, + "bert.encoder.layer.11.attention.self.value.weight": 589824, + "bert.encoder.layer.11.attention.self.value.bias": 768, + "bert.encoder.layer.11.attention.output.dense.weight": 589824, + "bert.encoder.layer.11.attention.output.dense.bias": 768, + "bert.encoder.layer.11.attention.output.LayerNorm.weight": 768, + "bert.encoder.layer.11.attention.output.LayerNorm.bias": 768, + "bert.encoder.layer.11.intermediate.dense.weight": 2359296, + "bert.encoder.layer.11.intermediate.dense.bias": 3072, + "bert.encoder.layer.11.output.dense.weight": 2359296, + "bert.encoder.layer.11.output.dense.bias": 768, + "bert.encoder.layer.11.output.LayerNorm.weight": 768, + "bert.encoder.layer.11.output.LayerNorm.bias": 768, + "feat_map.weight": 196608, + "feat_map.bias": 256, + "input_proj.0.0.weight": 65536, + "input_proj.0.0.bias": 256, + "input_proj.0.1.weight": 256, + "input_proj.0.1.bias": 256, + "input_proj.1.0.weight": 131072, + "input_proj.1.0.bias": 256, + "input_proj.1.1.weight": 256, + "input_proj.1.1.bias": 256, + "input_proj.2.0.weight": 262144, + "input_proj.2.0.bias": 256, + "input_proj.2.1.weight": 256, + "input_proj.2.1.bias": 256, + "input_proj.3.0.weight": 2359296, + "input_proj.3.0.bias": 256, + "input_proj.3.1.weight": 256, + "input_proj.3.1.bias": 256, + "backbone.0.patch_embed.proj.weight": 6144, + "backbone.0.patch_embed.proj.bias": 128, + "backbone.0.patch_embed.norm.weight": 128, + "backbone.0.patch_embed.norm.bias": 128, + "backbone.0.layers.0.blocks.0.norm1.weight": 128, + "backbone.0.layers.0.blocks.0.norm1.bias": 128, + "backbone.0.layers.0.blocks.0.attn.relative_position_bias_table": 2116, + "backbone.0.layers.0.blocks.0.attn.qkv.weight": 49152, + "backbone.0.layers.0.blocks.0.attn.qkv.bias": 384, + "backbone.0.layers.0.blocks.0.attn.proj.weight": 16384, + "backbone.0.layers.0.blocks.0.attn.proj.bias": 128, + "backbone.0.layers.0.blocks.0.norm2.weight": 128, + "backbone.0.layers.0.blocks.0.norm2.bias": 128, + "backbone.0.layers.0.blocks.0.mlp.fc1.weight": 65536, + "backbone.0.layers.0.blocks.0.mlp.fc1.bias": 512, + "backbone.0.layers.0.blocks.0.mlp.fc2.weight": 65536, + "backbone.0.layers.0.blocks.0.mlp.fc2.bias": 128, + "backbone.0.layers.0.blocks.1.norm1.weight": 128, + "backbone.0.layers.0.blocks.1.norm1.bias": 128, + "backbone.0.layers.0.blocks.1.attn.relative_position_bias_table": 2116, + "backbone.0.layers.0.blocks.1.attn.qkv.weight": 49152, + "backbone.0.layers.0.blocks.1.attn.qkv.bias": 384, + "backbone.0.layers.0.blocks.1.attn.proj.weight": 16384, + "backbone.0.layers.0.blocks.1.attn.proj.bias": 128, + "backbone.0.layers.0.blocks.1.norm2.weight": 128, + "backbone.0.layers.0.blocks.1.norm2.bias": 128, + "backbone.0.layers.0.blocks.1.mlp.fc1.weight": 65536, + "backbone.0.layers.0.blocks.1.mlp.fc1.bias": 512, + "backbone.0.layers.0.blocks.1.mlp.fc2.weight": 65536, + "backbone.0.layers.0.blocks.1.mlp.fc2.bias": 128, + "backbone.0.layers.0.downsample.reduction.weight": 131072, + "backbone.0.layers.0.downsample.norm.weight": 512, + "backbone.0.layers.0.downsample.norm.bias": 512, + "backbone.0.layers.1.blocks.0.norm1.weight": 256, + "backbone.0.layers.1.blocks.0.norm1.bias": 256, + "backbone.0.layers.1.blocks.0.attn.relative_position_bias_table": 4232, + "backbone.0.layers.1.blocks.0.attn.qkv.weight": 196608, + "backbone.0.layers.1.blocks.0.attn.qkv.bias": 768, + "backbone.0.layers.1.blocks.0.attn.proj.weight": 65536, + "backbone.0.layers.1.blocks.0.attn.proj.bias": 256, + "backbone.0.layers.1.blocks.0.norm2.weight": 256, + "backbone.0.layers.1.blocks.0.norm2.bias": 256, + "backbone.0.layers.1.blocks.0.mlp.fc1.weight": 262144, + "backbone.0.layers.1.blocks.0.mlp.fc1.bias": 1024, + "backbone.0.layers.1.blocks.0.mlp.fc2.weight": 262144, + "backbone.0.layers.1.blocks.0.mlp.fc2.bias": 256, + "backbone.0.layers.1.blocks.1.norm1.weight": 256, + "backbone.0.layers.1.blocks.1.norm1.bias": 256, + "backbone.0.layers.1.blocks.1.attn.relative_position_bias_table": 4232, + "backbone.0.layers.1.blocks.1.attn.qkv.weight": 196608, + "backbone.0.layers.1.blocks.1.attn.qkv.bias": 768, + "backbone.0.layers.1.blocks.1.attn.proj.weight": 65536, + "backbone.0.layers.1.blocks.1.attn.proj.bias": 256, + "backbone.0.layers.1.blocks.1.norm2.weight": 256, + "backbone.0.layers.1.blocks.1.norm2.bias": 256, + "backbone.0.layers.1.blocks.1.mlp.fc1.weight": 262144, + "backbone.0.layers.1.blocks.1.mlp.fc1.bias": 1024, + "backbone.0.layers.1.blocks.1.mlp.fc2.weight": 262144, + "backbone.0.layers.1.blocks.1.mlp.fc2.bias": 256, + "backbone.0.layers.1.downsample.reduction.weight": 524288, + "backbone.0.layers.1.downsample.norm.weight": 1024, + "backbone.0.layers.1.downsample.norm.bias": 1024, + "backbone.0.layers.2.blocks.0.norm1.weight": 512, + "backbone.0.layers.2.blocks.0.norm1.bias": 512, + "backbone.0.layers.2.blocks.0.attn.relative_position_bias_table": 8464, + "backbone.0.layers.2.blocks.0.attn.qkv.weight": 786432, + "backbone.0.layers.2.blocks.0.attn.qkv.bias": 1536, + "backbone.0.layers.2.blocks.0.attn.proj.weight": 262144, + "backbone.0.layers.2.blocks.0.attn.proj.bias": 512, + "backbone.0.layers.2.blocks.0.norm2.weight": 512, + "backbone.0.layers.2.blocks.0.norm2.bias": 512, + "backbone.0.layers.2.blocks.0.mlp.fc1.weight": 1048576, + "backbone.0.layers.2.blocks.0.mlp.fc1.bias": 2048, + "backbone.0.layers.2.blocks.0.mlp.fc2.weight": 1048576, + "backbone.0.layers.2.blocks.0.mlp.fc2.bias": 512, + "backbone.0.layers.2.blocks.1.norm1.weight": 512, + "backbone.0.layers.2.blocks.1.norm1.bias": 512, + "backbone.0.layers.2.blocks.1.attn.relative_position_bias_table": 8464, + "backbone.0.layers.2.blocks.1.attn.qkv.weight": 786432, + "backbone.0.layers.2.blocks.1.attn.qkv.bias": 1536, + "backbone.0.layers.2.blocks.1.attn.proj.weight": 262144, + "backbone.0.layers.2.blocks.1.attn.proj.bias": 512, + "backbone.0.layers.2.blocks.1.norm2.weight": 512, + "backbone.0.layers.2.blocks.1.norm2.bias": 512, + "backbone.0.layers.2.blocks.1.mlp.fc1.weight": 1048576, + "backbone.0.layers.2.blocks.1.mlp.fc1.bias": 2048, + "backbone.0.layers.2.blocks.1.mlp.fc2.weight": 1048576, + "backbone.0.layers.2.blocks.1.mlp.fc2.bias": 512, + "backbone.0.layers.2.blocks.2.norm1.weight": 512, + "backbone.0.layers.2.blocks.2.norm1.bias": 512, + "backbone.0.layers.2.blocks.2.attn.relative_position_bias_table": 8464, + "backbone.0.layers.2.blocks.2.attn.qkv.weight": 786432, + "backbone.0.layers.2.blocks.2.attn.qkv.bias": 1536, + "backbone.0.layers.2.blocks.2.attn.proj.weight": 262144, + "backbone.0.layers.2.blocks.2.attn.proj.bias": 512, + "backbone.0.layers.2.blocks.2.norm2.weight": 512, + "backbone.0.layers.2.blocks.2.norm2.bias": 512, + "backbone.0.layers.2.blocks.2.mlp.fc1.weight": 1048576, + "backbone.0.layers.2.blocks.2.mlp.fc1.bias": 2048, + "backbone.0.layers.2.blocks.2.mlp.fc2.weight": 1048576, + "backbone.0.layers.2.blocks.2.mlp.fc2.bias": 512, + "backbone.0.layers.2.blocks.3.norm1.weight": 512, + "backbone.0.layers.2.blocks.3.norm1.bias": 512, + "backbone.0.layers.2.blocks.3.attn.relative_position_bias_table": 8464, + "backbone.0.layers.2.blocks.3.attn.qkv.weight": 786432, + "backbone.0.layers.2.blocks.3.attn.qkv.bias": 1536, + "backbone.0.layers.2.blocks.3.attn.proj.weight": 262144, + "backbone.0.layers.2.blocks.3.attn.proj.bias": 512, + "backbone.0.layers.2.blocks.3.norm2.weight": 512, + "backbone.0.layers.2.blocks.3.norm2.bias": 512, + "backbone.0.layers.2.blocks.3.mlp.fc1.weight": 1048576, + "backbone.0.layers.2.blocks.3.mlp.fc1.bias": 2048, + "backbone.0.layers.2.blocks.3.mlp.fc2.weight": 1048576, + "backbone.0.layers.2.blocks.3.mlp.fc2.bias": 512, + "backbone.0.layers.2.blocks.4.norm1.weight": 512, + "backbone.0.layers.2.blocks.4.norm1.bias": 512, + "backbone.0.layers.2.blocks.4.attn.relative_position_bias_table": 8464, + "backbone.0.layers.2.blocks.4.attn.qkv.weight": 786432, + "backbone.0.layers.2.blocks.4.attn.qkv.bias": 1536, + "backbone.0.layers.2.blocks.4.attn.proj.weight": 262144, + "backbone.0.layers.2.blocks.4.attn.proj.bias": 512, + "backbone.0.layers.2.blocks.4.norm2.weight": 512, + "backbone.0.layers.2.blocks.4.norm2.bias": 512, + "backbone.0.layers.2.blocks.4.mlp.fc1.weight": 1048576, + "backbone.0.layers.2.blocks.4.mlp.fc1.bias": 2048, + "backbone.0.layers.2.blocks.4.mlp.fc2.weight": 1048576, + "backbone.0.layers.2.blocks.4.mlp.fc2.bias": 512, + "backbone.0.layers.2.blocks.5.norm1.weight": 512, + "backbone.0.layers.2.blocks.5.norm1.bias": 512, + "backbone.0.layers.2.blocks.5.attn.relative_position_bias_table": 8464, + "backbone.0.layers.2.blocks.5.attn.qkv.weight": 786432, + "backbone.0.layers.2.blocks.5.attn.qkv.bias": 1536, + "backbone.0.layers.2.blocks.5.attn.proj.weight": 262144, + "backbone.0.layers.2.blocks.5.attn.proj.bias": 512, + "backbone.0.layers.2.blocks.5.norm2.weight": 512, + "backbone.0.layers.2.blocks.5.norm2.bias": 512, + "backbone.0.layers.2.blocks.5.mlp.fc1.weight": 1048576, + "backbone.0.layers.2.blocks.5.mlp.fc1.bias": 2048, + "backbone.0.layers.2.blocks.5.mlp.fc2.weight": 1048576, + "backbone.0.layers.2.blocks.5.mlp.fc2.bias": 512, + "backbone.0.layers.2.blocks.6.norm1.weight": 512, + "backbone.0.layers.2.blocks.6.norm1.bias": 512, + "backbone.0.layers.2.blocks.6.attn.relative_position_bias_table": 8464, + "backbone.0.layers.2.blocks.6.attn.qkv.weight": 786432, + "backbone.0.layers.2.blocks.6.attn.qkv.bias": 1536, + "backbone.0.layers.2.blocks.6.attn.proj.weight": 262144, + "backbone.0.layers.2.blocks.6.attn.proj.bias": 512, + "backbone.0.layers.2.blocks.6.norm2.weight": 512, + "backbone.0.layers.2.blocks.6.norm2.bias": 512, + "backbone.0.layers.2.blocks.6.mlp.fc1.weight": 1048576, + "backbone.0.layers.2.blocks.6.mlp.fc1.bias": 2048, + "backbone.0.layers.2.blocks.6.mlp.fc2.weight": 1048576, + "backbone.0.layers.2.blocks.6.mlp.fc2.bias": 512, + "backbone.0.layers.2.blocks.7.norm1.weight": 512, + "backbone.0.layers.2.blocks.7.norm1.bias": 512, + "backbone.0.layers.2.blocks.7.attn.relative_position_bias_table": 8464, + "backbone.0.layers.2.blocks.7.attn.qkv.weight": 786432, + "backbone.0.layers.2.blocks.7.attn.qkv.bias": 1536, + "backbone.0.layers.2.blocks.7.attn.proj.weight": 262144, + "backbone.0.layers.2.blocks.7.attn.proj.bias": 512, + "backbone.0.layers.2.blocks.7.norm2.weight": 512, + "backbone.0.layers.2.blocks.7.norm2.bias": 512, + "backbone.0.layers.2.blocks.7.mlp.fc1.weight": 1048576, + "backbone.0.layers.2.blocks.7.mlp.fc1.bias": 2048, + "backbone.0.layers.2.blocks.7.mlp.fc2.weight": 1048576, + "backbone.0.layers.2.blocks.7.mlp.fc2.bias": 512, + "backbone.0.layers.2.blocks.8.norm1.weight": 512, + "backbone.0.layers.2.blocks.8.norm1.bias": 512, + "backbone.0.layers.2.blocks.8.attn.relative_position_bias_table": 8464, + "backbone.0.layers.2.blocks.8.attn.qkv.weight": 786432, + "backbone.0.layers.2.blocks.8.attn.qkv.bias": 1536, + "backbone.0.layers.2.blocks.8.attn.proj.weight": 262144, + "backbone.0.layers.2.blocks.8.attn.proj.bias": 512, + "backbone.0.layers.2.blocks.8.norm2.weight": 512, + "backbone.0.layers.2.blocks.8.norm2.bias": 512, + "backbone.0.layers.2.blocks.8.mlp.fc1.weight": 1048576, + "backbone.0.layers.2.blocks.8.mlp.fc1.bias": 2048, + "backbone.0.layers.2.blocks.8.mlp.fc2.weight": 1048576, + "backbone.0.layers.2.blocks.8.mlp.fc2.bias": 512, + "backbone.0.layers.2.blocks.9.norm1.weight": 512, + "backbone.0.layers.2.blocks.9.norm1.bias": 512, + "backbone.0.layers.2.blocks.9.attn.relative_position_bias_table": 8464, + "backbone.0.layers.2.blocks.9.attn.qkv.weight": 786432, + "backbone.0.layers.2.blocks.9.attn.qkv.bias": 1536, + "backbone.0.layers.2.blocks.9.attn.proj.weight": 262144, + "backbone.0.layers.2.blocks.9.attn.proj.bias": 512, + "backbone.0.layers.2.blocks.9.norm2.weight": 512, + "backbone.0.layers.2.blocks.9.norm2.bias": 512, + "backbone.0.layers.2.blocks.9.mlp.fc1.weight": 1048576, + "backbone.0.layers.2.blocks.9.mlp.fc1.bias": 2048, + "backbone.0.layers.2.blocks.9.mlp.fc2.weight": 1048576, + "backbone.0.layers.2.blocks.9.mlp.fc2.bias": 512, + "backbone.0.layers.2.blocks.10.norm1.weight": 512, + "backbone.0.layers.2.blocks.10.norm1.bias": 512, + "backbone.0.layers.2.blocks.10.attn.relative_position_bias_table": 8464, + "backbone.0.layers.2.blocks.10.attn.qkv.weight": 786432, + "backbone.0.layers.2.blocks.10.attn.qkv.bias": 1536, + "backbone.0.layers.2.blocks.10.attn.proj.weight": 262144, + "backbone.0.layers.2.blocks.10.attn.proj.bias": 512, + "backbone.0.layers.2.blocks.10.norm2.weight": 512, + "backbone.0.layers.2.blocks.10.norm2.bias": 512, + "backbone.0.layers.2.blocks.10.mlp.fc1.weight": 1048576, + "backbone.0.layers.2.blocks.10.mlp.fc1.bias": 2048, + "backbone.0.layers.2.blocks.10.mlp.fc2.weight": 1048576, + "backbone.0.layers.2.blocks.10.mlp.fc2.bias": 512, + "backbone.0.layers.2.blocks.11.norm1.weight": 512, + "backbone.0.layers.2.blocks.11.norm1.bias": 512, + "backbone.0.layers.2.blocks.11.attn.relative_position_bias_table": 8464, + "backbone.0.layers.2.blocks.11.attn.qkv.weight": 786432, + "backbone.0.layers.2.blocks.11.attn.qkv.bias": 1536, + "backbone.0.layers.2.blocks.11.attn.proj.weight": 262144, + "backbone.0.layers.2.blocks.11.attn.proj.bias": 512, + "backbone.0.layers.2.blocks.11.norm2.weight": 512, + "backbone.0.layers.2.blocks.11.norm2.bias": 512, + "backbone.0.layers.2.blocks.11.mlp.fc1.weight": 1048576, + "backbone.0.layers.2.blocks.11.mlp.fc1.bias": 2048, + "backbone.0.layers.2.blocks.11.mlp.fc2.weight": 1048576, + "backbone.0.layers.2.blocks.11.mlp.fc2.bias": 512, + "backbone.0.layers.2.blocks.12.norm1.weight": 512, + "backbone.0.layers.2.blocks.12.norm1.bias": 512, + "backbone.0.layers.2.blocks.12.attn.relative_position_bias_table": 8464, + "backbone.0.layers.2.blocks.12.attn.qkv.weight": 786432, + "backbone.0.layers.2.blocks.12.attn.qkv.bias": 1536, + "backbone.0.layers.2.blocks.12.attn.proj.weight": 262144, + "backbone.0.layers.2.blocks.12.attn.proj.bias": 512, + "backbone.0.layers.2.blocks.12.norm2.weight": 512, + "backbone.0.layers.2.blocks.12.norm2.bias": 512, + "backbone.0.layers.2.blocks.12.mlp.fc1.weight": 1048576, + "backbone.0.layers.2.blocks.12.mlp.fc1.bias": 2048, + "backbone.0.layers.2.blocks.12.mlp.fc2.weight": 1048576, + "backbone.0.layers.2.blocks.12.mlp.fc2.bias": 512, + "backbone.0.layers.2.blocks.13.norm1.weight": 512, + "backbone.0.layers.2.blocks.13.norm1.bias": 512, + "backbone.0.layers.2.blocks.13.attn.relative_position_bias_table": 8464, + "backbone.0.layers.2.blocks.13.attn.qkv.weight": 786432, + "backbone.0.layers.2.blocks.13.attn.qkv.bias": 1536, + "backbone.0.layers.2.blocks.13.attn.proj.weight": 262144, + "backbone.0.layers.2.blocks.13.attn.proj.bias": 512, + "backbone.0.layers.2.blocks.13.norm2.weight": 512, + "backbone.0.layers.2.blocks.13.norm2.bias": 512, + "backbone.0.layers.2.blocks.13.mlp.fc1.weight": 1048576, + "backbone.0.layers.2.blocks.13.mlp.fc1.bias": 2048, + "backbone.0.layers.2.blocks.13.mlp.fc2.weight": 1048576, + "backbone.0.layers.2.blocks.13.mlp.fc2.bias": 512, + "backbone.0.layers.2.blocks.14.norm1.weight": 512, + "backbone.0.layers.2.blocks.14.norm1.bias": 512, + "backbone.0.layers.2.blocks.14.attn.relative_position_bias_table": 8464, + "backbone.0.layers.2.blocks.14.attn.qkv.weight": 786432, + "backbone.0.layers.2.blocks.14.attn.qkv.bias": 1536, + "backbone.0.layers.2.blocks.14.attn.proj.weight": 262144, + "backbone.0.layers.2.blocks.14.attn.proj.bias": 512, + "backbone.0.layers.2.blocks.14.norm2.weight": 512, + "backbone.0.layers.2.blocks.14.norm2.bias": 512, + "backbone.0.layers.2.blocks.14.mlp.fc1.weight": 1048576, + "backbone.0.layers.2.blocks.14.mlp.fc1.bias": 2048, + "backbone.0.layers.2.blocks.14.mlp.fc2.weight": 1048576, + "backbone.0.layers.2.blocks.14.mlp.fc2.bias": 512, + "backbone.0.layers.2.blocks.15.norm1.weight": 512, + "backbone.0.layers.2.blocks.15.norm1.bias": 512, + "backbone.0.layers.2.blocks.15.attn.relative_position_bias_table": 8464, + "backbone.0.layers.2.blocks.15.attn.qkv.weight": 786432, + "backbone.0.layers.2.blocks.15.attn.qkv.bias": 1536, + "backbone.0.layers.2.blocks.15.attn.proj.weight": 262144, + "backbone.0.layers.2.blocks.15.attn.proj.bias": 512, + "backbone.0.layers.2.blocks.15.norm2.weight": 512, + "backbone.0.layers.2.blocks.15.norm2.bias": 512, + "backbone.0.layers.2.blocks.15.mlp.fc1.weight": 1048576, + "backbone.0.layers.2.blocks.15.mlp.fc1.bias": 2048, + "backbone.0.layers.2.blocks.15.mlp.fc2.weight": 1048576, + "backbone.0.layers.2.blocks.15.mlp.fc2.bias": 512, + "backbone.0.layers.2.blocks.16.norm1.weight": 512, + "backbone.0.layers.2.blocks.16.norm1.bias": 512, + "backbone.0.layers.2.blocks.16.attn.relative_position_bias_table": 8464, + "backbone.0.layers.2.blocks.16.attn.qkv.weight": 786432, + "backbone.0.layers.2.blocks.16.attn.qkv.bias": 1536, + "backbone.0.layers.2.blocks.16.attn.proj.weight": 262144, + "backbone.0.layers.2.blocks.16.attn.proj.bias": 512, + "backbone.0.layers.2.blocks.16.norm2.weight": 512, + "backbone.0.layers.2.blocks.16.norm2.bias": 512, + "backbone.0.layers.2.blocks.16.mlp.fc1.weight": 1048576, + "backbone.0.layers.2.blocks.16.mlp.fc1.bias": 2048, + "backbone.0.layers.2.blocks.16.mlp.fc2.weight": 1048576, + "backbone.0.layers.2.blocks.16.mlp.fc2.bias": 512, + "backbone.0.layers.2.blocks.17.norm1.weight": 512, + "backbone.0.layers.2.blocks.17.norm1.bias": 512, + "backbone.0.layers.2.blocks.17.attn.relative_position_bias_table": 8464, + "backbone.0.layers.2.blocks.17.attn.qkv.weight": 786432, + "backbone.0.layers.2.blocks.17.attn.qkv.bias": 1536, + "backbone.0.layers.2.blocks.17.attn.proj.weight": 262144, + "backbone.0.layers.2.blocks.17.attn.proj.bias": 512, + "backbone.0.layers.2.blocks.17.norm2.weight": 512, + "backbone.0.layers.2.blocks.17.norm2.bias": 512, + "backbone.0.layers.2.blocks.17.mlp.fc1.weight": 1048576, + "backbone.0.layers.2.blocks.17.mlp.fc1.bias": 2048, + "backbone.0.layers.2.blocks.17.mlp.fc2.weight": 1048576, + "backbone.0.layers.2.blocks.17.mlp.fc2.bias": 512, + "backbone.0.layers.2.downsample.reduction.weight": 2097152, + "backbone.0.layers.2.downsample.norm.weight": 2048, + "backbone.0.layers.2.downsample.norm.bias": 2048, + "backbone.0.layers.3.blocks.0.norm1.weight": 1024, + "backbone.0.layers.3.blocks.0.norm1.bias": 1024, + "backbone.0.layers.3.blocks.0.attn.relative_position_bias_table": 16928, + "backbone.0.layers.3.blocks.0.attn.qkv.weight": 3145728, + "backbone.0.layers.3.blocks.0.attn.qkv.bias": 3072, + "backbone.0.layers.3.blocks.0.attn.proj.weight": 1048576, + "backbone.0.layers.3.blocks.0.attn.proj.bias": 1024, + "backbone.0.layers.3.blocks.0.norm2.weight": 1024, + "backbone.0.layers.3.blocks.0.norm2.bias": 1024, + "backbone.0.layers.3.blocks.0.mlp.fc1.weight": 4194304, + "backbone.0.layers.3.blocks.0.mlp.fc1.bias": 4096, + "backbone.0.layers.3.blocks.0.mlp.fc2.weight": 4194304, + "backbone.0.layers.3.blocks.0.mlp.fc2.bias": 1024, + "backbone.0.layers.3.blocks.1.norm1.weight": 1024, + "backbone.0.layers.3.blocks.1.norm1.bias": 1024, + "backbone.0.layers.3.blocks.1.attn.relative_position_bias_table": 16928, + "backbone.0.layers.3.blocks.1.attn.qkv.weight": 3145728, + "backbone.0.layers.3.blocks.1.attn.qkv.bias": 3072, + "backbone.0.layers.3.blocks.1.attn.proj.weight": 1048576, + "backbone.0.layers.3.blocks.1.attn.proj.bias": 1024, + "backbone.0.layers.3.blocks.1.norm2.weight": 1024, + "backbone.0.layers.3.blocks.1.norm2.bias": 1024, + "backbone.0.layers.3.blocks.1.mlp.fc1.weight": 4194304, + "backbone.0.layers.3.blocks.1.mlp.fc1.bias": 4096, + "backbone.0.layers.3.blocks.1.mlp.fc2.weight": 4194304, + "backbone.0.layers.3.blocks.1.mlp.fc2.bias": 1024, + "backbone.0.norm1.weight": 256, + "backbone.0.norm1.bias": 256, + "backbone.0.norm2.weight": 512, + "backbone.0.norm2.bias": 512, + "backbone.0.norm3.weight": 1024, + "backbone.0.norm3.bias": 1024 +} +INFO  2024-03-29 17:46:28,634 | params after freezing: +{ + "transformer.level_embed": 1024, + "transformer.encoder.layers.0.self_attn.sampling_offsets.weight": 65536, + "transformer.encoder.layers.0.self_attn.sampling_offsets.bias": 256, + "transformer.encoder.layers.0.self_attn.attention_weights.weight": 32768, + "transformer.encoder.layers.0.self_attn.attention_weights.bias": 128, + "transformer.encoder.layers.0.self_attn.value_proj.weight": 65536, + "transformer.encoder.layers.0.self_attn.value_proj.bias": 256, + "transformer.encoder.layers.0.self_attn.output_proj.weight": 65536, + "transformer.encoder.layers.0.self_attn.output_proj.bias": 256, + "transformer.encoder.layers.0.norm1.weight": 256, + "transformer.encoder.layers.0.norm1.bias": 256, + "transformer.encoder.layers.0.linear1.weight": 524288, + "transformer.encoder.layers.0.linear1.bias": 2048, + "transformer.encoder.layers.0.linear2.weight": 524288, + "transformer.encoder.layers.0.linear2.bias": 256, + "transformer.encoder.layers.0.norm2.weight": 256, + "transformer.encoder.layers.0.norm2.bias": 256, + "transformer.encoder.layers.1.self_attn.sampling_offsets.weight": 65536, + "transformer.encoder.layers.1.self_attn.sampling_offsets.bias": 256, + "transformer.encoder.layers.1.self_attn.attention_weights.weight": 32768, + "transformer.encoder.layers.1.self_attn.attention_weights.bias": 128, + "transformer.encoder.layers.1.self_attn.value_proj.weight": 65536, + "transformer.encoder.layers.1.self_attn.value_proj.bias": 256, + "transformer.encoder.layers.1.self_attn.output_proj.weight": 65536, + "transformer.encoder.layers.1.self_attn.output_proj.bias": 256, + "transformer.encoder.layers.1.norm1.weight": 256, + "transformer.encoder.layers.1.norm1.bias": 256, + "transformer.encoder.layers.1.linear1.weight": 524288, + "transformer.encoder.layers.1.linear1.bias": 2048, + "transformer.encoder.layers.1.linear2.weight": 524288, + "transformer.encoder.layers.1.linear2.bias": 256, + "transformer.encoder.layers.1.norm2.weight": 256, + "transformer.encoder.layers.1.norm2.bias": 256, + "transformer.encoder.layers.2.self_attn.sampling_offsets.weight": 65536, + "transformer.encoder.layers.2.self_attn.sampling_offsets.bias": 256, + "transformer.encoder.layers.2.self_attn.attention_weights.weight": 32768, + "transformer.encoder.layers.2.self_attn.attention_weights.bias": 128, + "transformer.encoder.layers.2.self_attn.value_proj.weight": 65536, + "transformer.encoder.layers.2.self_attn.value_proj.bias": 256, + "transformer.encoder.layers.2.self_attn.output_proj.weight": 65536, + "transformer.encoder.layers.2.self_attn.output_proj.bias": 256, + "transformer.encoder.layers.2.norm1.weight": 256, + "transformer.encoder.layers.2.norm1.bias": 256, + "transformer.encoder.layers.2.linear1.weight": 524288, + "transformer.encoder.layers.2.linear1.bias": 2048, + "transformer.encoder.layers.2.linear2.weight": 524288, + "transformer.encoder.layers.2.linear2.bias": 256, + "transformer.encoder.layers.2.norm2.weight": 256, + "transformer.encoder.layers.2.norm2.bias": 256, + "transformer.encoder.layers.3.self_attn.sampling_offsets.weight": 65536, + "transformer.encoder.layers.3.self_attn.sampling_offsets.bias": 256, + "transformer.encoder.layers.3.self_attn.attention_weights.weight": 32768, + "transformer.encoder.layers.3.self_attn.attention_weights.bias": 128, + "transformer.encoder.layers.3.self_attn.value_proj.weight": 65536, + "transformer.encoder.layers.3.self_attn.value_proj.bias": 256, + "transformer.encoder.layers.3.self_attn.output_proj.weight": 65536, + "transformer.encoder.layers.3.self_attn.output_proj.bias": 256, + "transformer.encoder.layers.3.norm1.weight": 256, + "transformer.encoder.layers.3.norm1.bias": 256, + "transformer.encoder.layers.3.linear1.weight": 524288, + "transformer.encoder.layers.3.linear1.bias": 2048, + "transformer.encoder.layers.3.linear2.weight": 524288, + "transformer.encoder.layers.3.linear2.bias": 256, + "transformer.encoder.layers.3.norm2.weight": 256, + "transformer.encoder.layers.3.norm2.bias": 256, + "transformer.encoder.layers.4.self_attn.sampling_offsets.weight": 65536, + "transformer.encoder.layers.4.self_attn.sampling_offsets.bias": 256, + "transformer.encoder.layers.4.self_attn.attention_weights.weight": 32768, + "transformer.encoder.layers.4.self_attn.attention_weights.bias": 128, + "transformer.encoder.layers.4.self_attn.value_proj.weight": 65536, + "transformer.encoder.layers.4.self_attn.value_proj.bias": 256, + "transformer.encoder.layers.4.self_attn.output_proj.weight": 65536, + "transformer.encoder.layers.4.self_attn.output_proj.bias": 256, + "transformer.encoder.layers.4.norm1.weight": 256, + "transformer.encoder.layers.4.norm1.bias": 256, + "transformer.encoder.layers.4.linear1.weight": 524288, + "transformer.encoder.layers.4.linear1.bias": 2048, + "transformer.encoder.layers.4.linear2.weight": 524288, + "transformer.encoder.layers.4.linear2.bias": 256, + "transformer.encoder.layers.4.norm2.weight": 256, + "transformer.encoder.layers.4.norm2.bias": 256, + "transformer.encoder.layers.5.self_attn.sampling_offsets.weight": 65536, + "transformer.encoder.layers.5.self_attn.sampling_offsets.bias": 256, + "transformer.encoder.layers.5.self_attn.attention_weights.weight": 32768, + "transformer.encoder.layers.5.self_attn.attention_weights.bias": 128, + "transformer.encoder.layers.5.self_attn.value_proj.weight": 65536, + "transformer.encoder.layers.5.self_attn.value_proj.bias": 256, + "transformer.encoder.layers.5.self_attn.output_proj.weight": 65536, + "transformer.encoder.layers.5.self_attn.output_proj.bias": 256, + "transformer.encoder.layers.5.norm1.weight": 256, + "transformer.encoder.layers.5.norm1.bias": 256, + "transformer.encoder.layers.5.linear1.weight": 524288, + "transformer.encoder.layers.5.linear1.bias": 2048, + "transformer.encoder.layers.5.linear2.weight": 524288, + "transformer.encoder.layers.5.linear2.bias": 256, + "transformer.encoder.layers.5.norm2.weight": 256, + "transformer.encoder.layers.5.norm2.bias": 256, + "transformer.encoder.text_layers.0.self_attn.in_proj_weight": 196608, + "transformer.encoder.text_layers.0.self_attn.in_proj_bias": 768, + "transformer.encoder.text_layers.0.self_attn.out_proj.weight": 65536, + "transformer.encoder.text_layers.0.self_attn.out_proj.bias": 256, + "transformer.encoder.text_layers.0.linear1.weight": 262144, + "transformer.encoder.text_layers.0.linear1.bias": 1024, + "transformer.encoder.text_layers.0.linear2.weight": 262144, + "transformer.encoder.text_layers.0.linear2.bias": 256, + "transformer.encoder.text_layers.0.norm1.weight": 256, + "transformer.encoder.text_layers.0.norm1.bias": 256, + "transformer.encoder.text_layers.0.norm2.weight": 256, + "transformer.encoder.text_layers.0.norm2.bias": 256, + "transformer.encoder.text_layers.1.self_attn.in_proj_weight": 196608, + "transformer.encoder.text_layers.1.self_attn.in_proj_bias": 768, + "transformer.encoder.text_layers.1.self_attn.out_proj.weight": 65536, + "transformer.encoder.text_layers.1.self_attn.out_proj.bias": 256, + "transformer.encoder.text_layers.1.linear1.weight": 262144, + "transformer.encoder.text_layers.1.linear1.bias": 1024, + "transformer.encoder.text_layers.1.linear2.weight": 262144, + "transformer.encoder.text_layers.1.linear2.bias": 256, + "transformer.encoder.text_layers.1.norm1.weight": 256, + "transformer.encoder.text_layers.1.norm1.bias": 256, + "transformer.encoder.text_layers.1.norm2.weight": 256, + "transformer.encoder.text_layers.1.norm2.bias": 256, + "transformer.encoder.text_layers.2.self_attn.in_proj_weight": 196608, + "transformer.encoder.text_layers.2.self_attn.in_proj_bias": 768, + "transformer.encoder.text_layers.2.self_attn.out_proj.weight": 65536, + "transformer.encoder.text_layers.2.self_attn.out_proj.bias": 256, + "transformer.encoder.text_layers.2.linear1.weight": 262144, + "transformer.encoder.text_layers.2.linear1.bias": 1024, + "transformer.encoder.text_layers.2.linear2.weight": 262144, + "transformer.encoder.text_layers.2.linear2.bias": 256, + "transformer.encoder.text_layers.2.norm1.weight": 256, + "transformer.encoder.text_layers.2.norm1.bias": 256, + "transformer.encoder.text_layers.2.norm2.weight": 256, + "transformer.encoder.text_layers.2.norm2.bias": 256, + "transformer.encoder.text_layers.3.self_attn.in_proj_weight": 196608, + "transformer.encoder.text_layers.3.self_attn.in_proj_bias": 768, + "transformer.encoder.text_layers.3.self_attn.out_proj.weight": 65536, + "transformer.encoder.text_layers.3.self_attn.out_proj.bias": 256, + "transformer.encoder.text_layers.3.linear1.weight": 262144, + "transformer.encoder.text_layers.3.linear1.bias": 1024, + "transformer.encoder.text_layers.3.linear2.weight": 262144, + "transformer.encoder.text_layers.3.linear2.bias": 256, + "transformer.encoder.text_layers.3.norm1.weight": 256, + "transformer.encoder.text_layers.3.norm1.bias": 256, + "transformer.encoder.text_layers.3.norm2.weight": 256, + "transformer.encoder.text_layers.3.norm2.bias": 256, + "transformer.encoder.text_layers.4.self_attn.in_proj_weight": 196608, + "transformer.encoder.text_layers.4.self_attn.in_proj_bias": 768, + "transformer.encoder.text_layers.4.self_attn.out_proj.weight": 65536, + "transformer.encoder.text_layers.4.self_attn.out_proj.bias": 256, + "transformer.encoder.text_layers.4.linear1.weight": 262144, + "transformer.encoder.text_layers.4.linear1.bias": 1024, + "transformer.encoder.text_layers.4.linear2.weight": 262144, + "transformer.encoder.text_layers.4.linear2.bias": 256, + "transformer.encoder.text_layers.4.norm1.weight": 256, + "transformer.encoder.text_layers.4.norm1.bias": 256, + "transformer.encoder.text_layers.4.norm2.weight": 256, + "transformer.encoder.text_layers.4.norm2.bias": 256, + "transformer.encoder.text_layers.5.self_attn.in_proj_weight": 196608, + "transformer.encoder.text_layers.5.self_attn.in_proj_bias": 768, + "transformer.encoder.text_layers.5.self_attn.out_proj.weight": 65536, + "transformer.encoder.text_layers.5.self_attn.out_proj.bias": 256, + "transformer.encoder.text_layers.5.linear1.weight": 262144, + "transformer.encoder.text_layers.5.linear1.bias": 1024, + "transformer.encoder.text_layers.5.linear2.weight": 262144, + "transformer.encoder.text_layers.5.linear2.bias": 256, + "transformer.encoder.text_layers.5.norm1.weight": 256, + "transformer.encoder.text_layers.5.norm1.bias": 256, + "transformer.encoder.text_layers.5.norm2.weight": 256, + "transformer.encoder.text_layers.5.norm2.bias": 256, + "transformer.encoder.fusion_layers.0.gamma_v": 256, + "transformer.encoder.fusion_layers.0.gamma_l": 256, + "transformer.encoder.fusion_layers.0.layer_norm_v.weight": 256, + "transformer.encoder.fusion_layers.0.layer_norm_v.bias": 256, + "transformer.encoder.fusion_layers.0.layer_norm_l.weight": 256, + "transformer.encoder.fusion_layers.0.layer_norm_l.bias": 256, + "transformer.encoder.fusion_layers.0.attn.v_proj.weight": 262144, + "transformer.encoder.fusion_layers.0.attn.v_proj.bias": 1024, + "transformer.encoder.fusion_layers.0.attn.l_proj.weight": 262144, + "transformer.encoder.fusion_layers.0.attn.l_proj.bias": 1024, + "transformer.encoder.fusion_layers.0.attn.values_v_proj.weight": 262144, + "transformer.encoder.fusion_layers.0.attn.values_v_proj.bias": 1024, + "transformer.encoder.fusion_layers.0.attn.values_l_proj.weight": 262144, + "transformer.encoder.fusion_layers.0.attn.values_l_proj.bias": 1024, + "transformer.encoder.fusion_layers.0.attn.out_v_proj.weight": 262144, + "transformer.encoder.fusion_layers.0.attn.out_v_proj.bias": 256, + "transformer.encoder.fusion_layers.0.attn.out_l_proj.weight": 262144, + "transformer.encoder.fusion_layers.0.attn.out_l_proj.bias": 256, + "transformer.encoder.fusion_layers.1.gamma_v": 256, + "transformer.encoder.fusion_layers.1.gamma_l": 256, + "transformer.encoder.fusion_layers.1.layer_norm_v.weight": 256, + "transformer.encoder.fusion_layers.1.layer_norm_v.bias": 256, + "transformer.encoder.fusion_layers.1.layer_norm_l.weight": 256, + "transformer.encoder.fusion_layers.1.layer_norm_l.bias": 256, + "transformer.encoder.fusion_layers.1.attn.v_proj.weight": 262144, + "transformer.encoder.fusion_layers.1.attn.v_proj.bias": 1024, + "transformer.encoder.fusion_layers.1.attn.l_proj.weight": 262144, + "transformer.encoder.fusion_layers.1.attn.l_proj.bias": 1024, + "transformer.encoder.fusion_layers.1.attn.values_v_proj.weight": 262144, + "transformer.encoder.fusion_layers.1.attn.values_v_proj.bias": 1024, + "transformer.encoder.fusion_layers.1.attn.values_l_proj.weight": 262144, + "transformer.encoder.fusion_layers.1.attn.values_l_proj.bias": 1024, + "transformer.encoder.fusion_layers.1.attn.out_v_proj.weight": 262144, + "transformer.encoder.fusion_layers.1.attn.out_v_proj.bias": 256, + "transformer.encoder.fusion_layers.1.attn.out_l_proj.weight": 262144, + "transformer.encoder.fusion_layers.1.attn.out_l_proj.bias": 256, + "transformer.encoder.fusion_layers.2.gamma_v": 256, + "transformer.encoder.fusion_layers.2.gamma_l": 256, + "transformer.encoder.fusion_layers.2.layer_norm_v.weight": 256, + "transformer.encoder.fusion_layers.2.layer_norm_v.bias": 256, + "transformer.encoder.fusion_layers.2.layer_norm_l.weight": 256, + "transformer.encoder.fusion_layers.2.layer_norm_l.bias": 256, + "transformer.encoder.fusion_layers.2.attn.v_proj.weight": 262144, + "transformer.encoder.fusion_layers.2.attn.v_proj.bias": 1024, + "transformer.encoder.fusion_layers.2.attn.l_proj.weight": 262144, + "transformer.encoder.fusion_layers.2.attn.l_proj.bias": 1024, + "transformer.encoder.fusion_layers.2.attn.values_v_proj.weight": 262144, + "transformer.encoder.fusion_layers.2.attn.values_v_proj.bias": 1024, + "transformer.encoder.fusion_layers.2.attn.values_l_proj.weight": 262144, + "transformer.encoder.fusion_layers.2.attn.values_l_proj.bias": 1024, + "transformer.encoder.fusion_layers.2.attn.out_v_proj.weight": 262144, + "transformer.encoder.fusion_layers.2.attn.out_v_proj.bias": 256, + "transformer.encoder.fusion_layers.2.attn.out_l_proj.weight": 262144, + "transformer.encoder.fusion_layers.2.attn.out_l_proj.bias": 256, + "transformer.encoder.fusion_layers.3.gamma_v": 256, + "transformer.encoder.fusion_layers.3.gamma_l": 256, + "transformer.encoder.fusion_layers.3.layer_norm_v.weight": 256, + "transformer.encoder.fusion_layers.3.layer_norm_v.bias": 256, + "transformer.encoder.fusion_layers.3.layer_norm_l.weight": 256, + "transformer.encoder.fusion_layers.3.layer_norm_l.bias": 256, + "transformer.encoder.fusion_layers.3.attn.v_proj.weight": 262144, + "transformer.encoder.fusion_layers.3.attn.v_proj.bias": 1024, + "transformer.encoder.fusion_layers.3.attn.l_proj.weight": 262144, + "transformer.encoder.fusion_layers.3.attn.l_proj.bias": 1024, + "transformer.encoder.fusion_layers.3.attn.values_v_proj.weight": 262144, + "transformer.encoder.fusion_layers.3.attn.values_v_proj.bias": 1024, + "transformer.encoder.fusion_layers.3.attn.values_l_proj.weight": 262144, + "transformer.encoder.fusion_layers.3.attn.values_l_proj.bias": 1024, + "transformer.encoder.fusion_layers.3.attn.out_v_proj.weight": 262144, + "transformer.encoder.fusion_layers.3.attn.out_v_proj.bias": 256, + "transformer.encoder.fusion_layers.3.attn.out_l_proj.weight": 262144, + "transformer.encoder.fusion_layers.3.attn.out_l_proj.bias": 256, + "transformer.encoder.fusion_layers.4.gamma_v": 256, + "transformer.encoder.fusion_layers.4.gamma_l": 256, + "transformer.encoder.fusion_layers.4.layer_norm_v.weight": 256, + "transformer.encoder.fusion_layers.4.layer_norm_v.bias": 256, + "transformer.encoder.fusion_layers.4.layer_norm_l.weight": 256, + "transformer.encoder.fusion_layers.4.layer_norm_l.bias": 256, + "transformer.encoder.fusion_layers.4.attn.v_proj.weight": 262144, + "transformer.encoder.fusion_layers.4.attn.v_proj.bias": 1024, + "transformer.encoder.fusion_layers.4.attn.l_proj.weight": 262144, + "transformer.encoder.fusion_layers.4.attn.l_proj.bias": 1024, + "transformer.encoder.fusion_layers.4.attn.values_v_proj.weight": 262144, + "transformer.encoder.fusion_layers.4.attn.values_v_proj.bias": 1024, + "transformer.encoder.fusion_layers.4.attn.values_l_proj.weight": 262144, + "transformer.encoder.fusion_layers.4.attn.values_l_proj.bias": 1024, + "transformer.encoder.fusion_layers.4.attn.out_v_proj.weight": 262144, + "transformer.encoder.fusion_layers.4.attn.out_v_proj.bias": 256, + "transformer.encoder.fusion_layers.4.attn.out_l_proj.weight": 262144, + "transformer.encoder.fusion_layers.4.attn.out_l_proj.bias": 256, + "transformer.encoder.fusion_layers.5.gamma_v": 256, + "transformer.encoder.fusion_layers.5.gamma_l": 256, + "transformer.encoder.fusion_layers.5.layer_norm_v.weight": 256, + "transformer.encoder.fusion_layers.5.layer_norm_v.bias": 256, + "transformer.encoder.fusion_layers.5.layer_norm_l.weight": 256, + "transformer.encoder.fusion_layers.5.layer_norm_l.bias": 256, + "transformer.encoder.fusion_layers.5.attn.v_proj.weight": 262144, + "transformer.encoder.fusion_layers.5.attn.v_proj.bias": 1024, + "transformer.encoder.fusion_layers.5.attn.l_proj.weight": 262144, + "transformer.encoder.fusion_layers.5.attn.l_proj.bias": 1024, + "transformer.encoder.fusion_layers.5.attn.values_v_proj.weight": 262144, + "transformer.encoder.fusion_layers.5.attn.values_v_proj.bias": 1024, + "transformer.encoder.fusion_layers.5.attn.values_l_proj.weight": 262144, + "transformer.encoder.fusion_layers.5.attn.values_l_proj.bias": 1024, + "transformer.encoder.fusion_layers.5.attn.out_v_proj.weight": 262144, + "transformer.encoder.fusion_layers.5.attn.out_v_proj.bias": 256, + "transformer.encoder.fusion_layers.5.attn.out_l_proj.weight": 262144, + "transformer.encoder.fusion_layers.5.attn.out_l_proj.bias": 256, + "transformer.decoder.layers.0.cross_attn.sampling_offsets.weight": 65536, + "transformer.decoder.layers.0.cross_attn.sampling_offsets.bias": 256, + "transformer.decoder.layers.0.cross_attn.attention_weights.weight": 32768, + "transformer.decoder.layers.0.cross_attn.attention_weights.bias": 128, + "transformer.decoder.layers.0.cross_attn.value_proj.weight": 65536, + "transformer.decoder.layers.0.cross_attn.value_proj.bias": 256, + "transformer.decoder.layers.0.cross_attn.output_proj.weight": 65536, + "transformer.decoder.layers.0.cross_attn.output_proj.bias": 256, + "transformer.decoder.layers.0.norm1.weight": 256, + "transformer.decoder.layers.0.norm1.bias": 256, + "transformer.decoder.layers.0.ca_text.in_proj_weight": 196608, + "transformer.decoder.layers.0.ca_text.in_proj_bias": 768, + "transformer.decoder.layers.0.ca_text.out_proj.weight": 65536, + "transformer.decoder.layers.0.ca_text.out_proj.bias": 256, + "transformer.decoder.layers.0.catext_norm.weight": 256, + "transformer.decoder.layers.0.catext_norm.bias": 256, + "transformer.decoder.layers.0.self_attn.in_proj_weight": 196608, + "transformer.decoder.layers.0.self_attn.in_proj_bias": 768, + "transformer.decoder.layers.0.self_attn.out_proj.weight": 65536, + "transformer.decoder.layers.0.self_attn.out_proj.bias": 256, + "transformer.decoder.layers.0.norm2.weight": 256, + "transformer.decoder.layers.0.norm2.bias": 256, + "transformer.decoder.layers.0.linear1.weight": 524288, + "transformer.decoder.layers.0.linear1.bias": 2048, + "transformer.decoder.layers.0.linear2.weight": 524288, + "transformer.decoder.layers.0.linear2.bias": 256, + "transformer.decoder.layers.0.norm3.weight": 256, + "transformer.decoder.layers.0.norm3.bias": 256, + "transformer.decoder.layers.1.cross_attn.sampling_offsets.weight": 65536, + "transformer.decoder.layers.1.cross_attn.sampling_offsets.bias": 256, + "transformer.decoder.layers.1.cross_attn.attention_weights.weight": 32768, + "transformer.decoder.layers.1.cross_attn.attention_weights.bias": 128, + "transformer.decoder.layers.1.cross_attn.value_proj.weight": 65536, + "transformer.decoder.layers.1.cross_attn.value_proj.bias": 256, + "transformer.decoder.layers.1.cross_attn.output_proj.weight": 65536, + "transformer.decoder.layers.1.cross_attn.output_proj.bias": 256, + "transformer.decoder.layers.1.norm1.weight": 256, + "transformer.decoder.layers.1.norm1.bias": 256, + "transformer.decoder.layers.1.ca_text.in_proj_weight": 196608, + "transformer.decoder.layers.1.ca_text.in_proj_bias": 768, + "transformer.decoder.layers.1.ca_text.out_proj.weight": 65536, + "transformer.decoder.layers.1.ca_text.out_proj.bias": 256, + "transformer.decoder.layers.1.catext_norm.weight": 256, + "transformer.decoder.layers.1.catext_norm.bias": 256, + "transformer.decoder.layers.1.self_attn.in_proj_weight": 196608, + "transformer.decoder.layers.1.self_attn.in_proj_bias": 768, + "transformer.decoder.layers.1.self_attn.out_proj.weight": 65536, + "transformer.decoder.layers.1.self_attn.out_proj.bias": 256, + "transformer.decoder.layers.1.norm2.weight": 256, + "transformer.decoder.layers.1.norm2.bias": 256, + "transformer.decoder.layers.1.linear1.weight": 524288, + "transformer.decoder.layers.1.linear1.bias": 2048, + "transformer.decoder.layers.1.linear2.weight": 524288, + "transformer.decoder.layers.1.linear2.bias": 256, + "transformer.decoder.layers.1.norm3.weight": 256, + "transformer.decoder.layers.1.norm3.bias": 256, + "transformer.decoder.layers.2.cross_attn.sampling_offsets.weight": 65536, + "transformer.decoder.layers.2.cross_attn.sampling_offsets.bias": 256, + "transformer.decoder.layers.2.cross_attn.attention_weights.weight": 32768, + "transformer.decoder.layers.2.cross_attn.attention_weights.bias": 128, + "transformer.decoder.layers.2.cross_attn.value_proj.weight": 65536, + "transformer.decoder.layers.2.cross_attn.value_proj.bias": 256, + "transformer.decoder.layers.2.cross_attn.output_proj.weight": 65536, + "transformer.decoder.layers.2.cross_attn.output_proj.bias": 256, + "transformer.decoder.layers.2.norm1.weight": 256, + "transformer.decoder.layers.2.norm1.bias": 256, + "transformer.decoder.layers.2.ca_text.in_proj_weight": 196608, + "transformer.decoder.layers.2.ca_text.in_proj_bias": 768, + "transformer.decoder.layers.2.ca_text.out_proj.weight": 65536, + "transformer.decoder.layers.2.ca_text.out_proj.bias": 256, + "transformer.decoder.layers.2.catext_norm.weight": 256, + "transformer.decoder.layers.2.catext_norm.bias": 256, + "transformer.decoder.layers.2.self_attn.in_proj_weight": 196608, + "transformer.decoder.layers.2.self_attn.in_proj_bias": 768, + "transformer.decoder.layers.2.self_attn.out_proj.weight": 65536, + "transformer.decoder.layers.2.self_attn.out_proj.bias": 256, + "transformer.decoder.layers.2.norm2.weight": 256, + "transformer.decoder.layers.2.norm2.bias": 256, + "transformer.decoder.layers.2.linear1.weight": 524288, + "transformer.decoder.layers.2.linear1.bias": 2048, + "transformer.decoder.layers.2.linear2.weight": 524288, + "transformer.decoder.layers.2.linear2.bias": 256, + "transformer.decoder.layers.2.norm3.weight": 256, + "transformer.decoder.layers.2.norm3.bias": 256, + "transformer.decoder.layers.3.cross_attn.sampling_offsets.weight": 65536, + "transformer.decoder.layers.3.cross_attn.sampling_offsets.bias": 256, + "transformer.decoder.layers.3.cross_attn.attention_weights.weight": 32768, + "transformer.decoder.layers.3.cross_attn.attention_weights.bias": 128, + "transformer.decoder.layers.3.cross_attn.value_proj.weight": 65536, + "transformer.decoder.layers.3.cross_attn.value_proj.bias": 256, + "transformer.decoder.layers.3.cross_attn.output_proj.weight": 65536, + "transformer.decoder.layers.3.cross_attn.output_proj.bias": 256, + "transformer.decoder.layers.3.norm1.weight": 256, + "transformer.decoder.layers.3.norm1.bias": 256, + "transformer.decoder.layers.3.ca_text.in_proj_weight": 196608, + "transformer.decoder.layers.3.ca_text.in_proj_bias": 768, + "transformer.decoder.layers.3.ca_text.out_proj.weight": 65536, + "transformer.decoder.layers.3.ca_text.out_proj.bias": 256, + "transformer.decoder.layers.3.catext_norm.weight": 256, + "transformer.decoder.layers.3.catext_norm.bias": 256, + "transformer.decoder.layers.3.self_attn.in_proj_weight": 196608, + "transformer.decoder.layers.3.self_attn.in_proj_bias": 768, + "transformer.decoder.layers.3.self_attn.out_proj.weight": 65536, + "transformer.decoder.layers.3.self_attn.out_proj.bias": 256, + "transformer.decoder.layers.3.norm2.weight": 256, + "transformer.decoder.layers.3.norm2.bias": 256, + "transformer.decoder.layers.3.linear1.weight": 524288, + "transformer.decoder.layers.3.linear1.bias": 2048, + "transformer.decoder.layers.3.linear2.weight": 524288, + "transformer.decoder.layers.3.linear2.bias": 256, + "transformer.decoder.layers.3.norm3.weight": 256, + "transformer.decoder.layers.3.norm3.bias": 256, + "transformer.decoder.layers.4.cross_attn.sampling_offsets.weight": 65536, + "transformer.decoder.layers.4.cross_attn.sampling_offsets.bias": 256, + "transformer.decoder.layers.4.cross_attn.attention_weights.weight": 32768, + "transformer.decoder.layers.4.cross_attn.attention_weights.bias": 128, + "transformer.decoder.layers.4.cross_attn.value_proj.weight": 65536, + "transformer.decoder.layers.4.cross_attn.value_proj.bias": 256, + "transformer.decoder.layers.4.cross_attn.output_proj.weight": 65536, + "transformer.decoder.layers.4.cross_attn.output_proj.bias": 256, + "transformer.decoder.layers.4.norm1.weight": 256, + "transformer.decoder.layers.4.norm1.bias": 256, + "transformer.decoder.layers.4.ca_text.in_proj_weight": 196608, + "transformer.decoder.layers.4.ca_text.in_proj_bias": 768, + "transformer.decoder.layers.4.ca_text.out_proj.weight": 65536, + "transformer.decoder.layers.4.ca_text.out_proj.bias": 256, + "transformer.decoder.layers.4.catext_norm.weight": 256, + "transformer.decoder.layers.4.catext_norm.bias": 256, + "transformer.decoder.layers.4.self_attn.in_proj_weight": 196608, + "transformer.decoder.layers.4.self_attn.in_proj_bias": 768, + "transformer.decoder.layers.4.self_attn.out_proj.weight": 65536, + "transformer.decoder.layers.4.self_attn.out_proj.bias": 256, + "transformer.decoder.layers.4.norm2.weight": 256, + "transformer.decoder.layers.4.norm2.bias": 256, + "transformer.decoder.layers.4.linear1.weight": 524288, + "transformer.decoder.layers.4.linear1.bias": 2048, + "transformer.decoder.layers.4.linear2.weight": 524288, + "transformer.decoder.layers.4.linear2.bias": 256, + "transformer.decoder.layers.4.norm3.weight": 256, + "transformer.decoder.layers.4.norm3.bias": 256, + "transformer.decoder.layers.5.cross_attn.sampling_offsets.weight": 65536, + "transformer.decoder.layers.5.cross_attn.sampling_offsets.bias": 256, + "transformer.decoder.layers.5.cross_attn.attention_weights.weight": 32768, + "transformer.decoder.layers.5.cross_attn.attention_weights.bias": 128, + "transformer.decoder.layers.5.cross_attn.value_proj.weight": 65536, + "transformer.decoder.layers.5.cross_attn.value_proj.bias": 256, + "transformer.decoder.layers.5.cross_attn.output_proj.weight": 65536, + "transformer.decoder.layers.5.cross_attn.output_proj.bias": 256, + "transformer.decoder.layers.5.norm1.weight": 256, + "transformer.decoder.layers.5.norm1.bias": 256, + "transformer.decoder.layers.5.ca_text.in_proj_weight": 196608, + "transformer.decoder.layers.5.ca_text.in_proj_bias": 768, + "transformer.decoder.layers.5.ca_text.out_proj.weight": 65536, + "transformer.decoder.layers.5.ca_text.out_proj.bias": 256, + "transformer.decoder.layers.5.catext_norm.weight": 256, + "transformer.decoder.layers.5.catext_norm.bias": 256, + "transformer.decoder.layers.5.self_attn.in_proj_weight": 196608, + "transformer.decoder.layers.5.self_attn.in_proj_bias": 768, + "transformer.decoder.layers.5.self_attn.out_proj.weight": 65536, + "transformer.decoder.layers.5.self_attn.out_proj.bias": 256, + "transformer.decoder.layers.5.norm2.weight": 256, + "transformer.decoder.layers.5.norm2.bias": 256, + "transformer.decoder.layers.5.linear1.weight": 524288, + "transformer.decoder.layers.5.linear1.bias": 2048, + "transformer.decoder.layers.5.linear2.weight": 524288, + "transformer.decoder.layers.5.linear2.bias": 256, + "transformer.decoder.layers.5.norm3.weight": 256, + "transformer.decoder.layers.5.norm3.bias": 256, + "transformer.decoder.norm.weight": 256, + "transformer.decoder.norm.bias": 256, + "transformer.decoder.ref_point_head.layers.0.weight": 131072, + "transformer.decoder.ref_point_head.layers.0.bias": 256, + "transformer.decoder.ref_point_head.layers.1.weight": 65536, + "transformer.decoder.ref_point_head.layers.1.bias": 256, + "transformer.decoder.bbox_embed.0.layers.0.weight": 65536, + "transformer.decoder.bbox_embed.0.layers.0.bias": 256, + "transformer.decoder.bbox_embed.0.layers.1.weight": 65536, + "transformer.decoder.bbox_embed.0.layers.1.bias": 256, + "transformer.decoder.bbox_embed.0.layers.2.weight": 1024, + "transformer.decoder.bbox_embed.0.layers.2.bias": 4, + "transformer.tgt_embed.weight": 230400, + "transformer.enc_output.weight": 65536, + "transformer.enc_output.bias": 256, + "transformer.enc_output_norm.weight": 256, + "transformer.enc_output_norm.bias": 256, + "transformer.enc_out_bbox_embed.layers.0.weight": 65536, + "transformer.enc_out_bbox_embed.layers.0.bias": 256, + "transformer.enc_out_bbox_embed.layers.1.weight": 65536, + "transformer.enc_out_bbox_embed.layers.1.bias": 256, + "transformer.enc_out_bbox_embed.layers.2.weight": 1024, + "transformer.enc_out_bbox_embed.layers.2.bias": 4, + "feature_map_proj.weight": 458752, + "feature_map_proj.bias": 256, + "feature_map_encoder.layers.0.norm1.weight": 256, + "feature_map_encoder.layers.0.norm1.bias": 256, + "feature_map_encoder.layers.0.norm2.weight": 256, + "feature_map_encoder.layers.0.norm2.bias": 256, + "feature_map_encoder.layers.0.self_attn.in_proj_weight": 196608, + "feature_map_encoder.layers.0.self_attn.in_proj_bias": 768, + "feature_map_encoder.layers.0.self_attn.out_proj.weight": 65536, + "feature_map_encoder.layers.0.self_attn.out_proj.bias": 256, + "feature_map_encoder.layers.0.mlp.linear1.weight": 524288, + "feature_map_encoder.layers.0.mlp.linear1.bias": 2048, + "feature_map_encoder.layers.0.mlp.linear2.weight": 524288, + "feature_map_encoder.layers.0.mlp.linear2.bias": 256, + "feature_map_encoder.layers.1.norm1.weight": 256, + "feature_map_encoder.layers.1.norm1.bias": 256, + "feature_map_encoder.layers.1.norm2.weight": 256, + "feature_map_encoder.layers.1.norm2.bias": 256, + "feature_map_encoder.layers.1.self_attn.in_proj_weight": 196608, + "feature_map_encoder.layers.1.self_attn.in_proj_bias": 768, + "feature_map_encoder.layers.1.self_attn.out_proj.weight": 65536, + "feature_map_encoder.layers.1.self_attn.out_proj.bias": 256, + "feature_map_encoder.layers.1.mlp.linear1.weight": 524288, + "feature_map_encoder.layers.1.mlp.linear1.bias": 2048, + "feature_map_encoder.layers.1.mlp.linear2.weight": 524288, + "feature_map_encoder.layers.1.mlp.linear2.bias": 256, + "feature_map_encoder.layers.2.norm1.weight": 256, + "feature_map_encoder.layers.2.norm1.bias": 256, + "feature_map_encoder.layers.2.norm2.weight": 256, + "feature_map_encoder.layers.2.norm2.bias": 256, + "feature_map_encoder.layers.2.self_attn.in_proj_weight": 196608, + "feature_map_encoder.layers.2.self_attn.in_proj_bias": 768, + "feature_map_encoder.layers.2.self_attn.out_proj.weight": 65536, + "feature_map_encoder.layers.2.self_attn.out_proj.bias": 256, + "feature_map_encoder.layers.2.mlp.linear1.weight": 524288, + "feature_map_encoder.layers.2.mlp.linear1.bias": 2048, + "feature_map_encoder.layers.2.mlp.linear2.weight": 524288, + "feature_map_encoder.layers.2.mlp.linear2.bias": 256, + "feature_map_encoder.norm.weight": 256, + "feature_map_encoder.norm.bias": 256, + "feat_map.weight": 196608, + "feat_map.bias": 256, + "input_proj.0.0.weight": 65536, + "input_proj.0.0.bias": 256, + "input_proj.0.1.weight": 256, + "input_proj.0.1.bias": 256, + "input_proj.1.0.weight": 131072, + "input_proj.1.0.bias": 256, + "input_proj.1.1.weight": 256, + "input_proj.1.1.bias": 256, + "input_proj.2.0.weight": 262144, + "input_proj.2.0.bias": 256, + "input_proj.2.1.weight": 256, + "input_proj.2.1.bias": 256, + "input_proj.3.0.weight": 2359296, + "input_proj.3.0.bias": 256, + "input_proj.3.1.weight": 256, + "input_proj.3.1.bias": 256 +} +DEBUG  2024-03-29 17:46:28,636 | build dataset ... ... +DEBUG  2024-03-29 17:46:29,266 | build dataset, done. +DEBUG  2024-03-29 17:46:29,267 | number of training dataset: 1, samples: 3659 +INFO  2024-03-29 17:46:29,889 | Ignore keys: [] +INFO  2024-03-29 17:46:30,040 | _IncompatibleKeys(missing_keys=['feature_map_proj.weight', 'feature_map_proj.bias', 'feature_map_encoder.layers.0.norm1.weight', 'feature_map_encoder.layers.0.norm1.bias', 'feature_map_encoder.layers.0.norm2.weight', 'feature_map_encoder.layers.0.norm2.bias', 'feature_map_encoder.layers.0.self_attn.in_proj_weight', 'feature_map_encoder.layers.0.self_attn.in_proj_bias', 'feature_map_encoder.layers.0.self_attn.out_proj.weight', 'feature_map_encoder.layers.0.self_attn.out_proj.bias', 'feature_map_encoder.layers.0.mlp.linear1.weight', 'feature_map_encoder.layers.0.mlp.linear1.bias', 'feature_map_encoder.layers.0.mlp.linear2.weight', 'feature_map_encoder.layers.0.mlp.linear2.bias', 'feature_map_encoder.layers.1.norm1.weight', 'feature_map_encoder.layers.1.norm1.bias', 'feature_map_encoder.layers.1.norm2.weight', 'feature_map_encoder.layers.1.norm2.bias', 'feature_map_encoder.layers.1.self_attn.in_proj_weight', 'feature_map_encoder.layers.1.self_attn.in_proj_bias', 'feature_map_encoder.layers.1.self_attn.out_proj.weight', 'feature_map_encoder.layers.1.self_attn.out_proj.bias', 'feature_map_encoder.layers.1.mlp.linear1.weight', 'feature_map_encoder.layers.1.mlp.linear1.bias', 'feature_map_encoder.layers.1.mlp.linear2.weight', 'feature_map_encoder.layers.1.mlp.linear2.bias', 'feature_map_encoder.layers.2.norm1.weight', 'feature_map_encoder.layers.2.norm1.bias', 'feature_map_encoder.layers.2.norm2.weight', 'feature_map_encoder.layers.2.norm2.bias', 'feature_map_encoder.layers.2.self_attn.in_proj_weight', 'feature_map_encoder.layers.2.self_attn.in_proj_bias', 'feature_map_encoder.layers.2.self_attn.out_proj.weight', 'feature_map_encoder.layers.2.self_attn.out_proj.bias', 'feature_map_encoder.layers.2.mlp.linear1.weight', 'feature_map_encoder.layers.2.mlp.linear1.bias', 'feature_map_encoder.layers.2.mlp.linear2.weight', 'feature_map_encoder.layers.2.mlp.linear2.bias', 'feature_map_encoder.norm.weight', 'feature_map_encoder.norm.bias'], unexpected_keys=['label_enc.weight', 'bert.embeddings.position_ids']) diff --git a/deer.jpg b/deer.jpg new file mode 100644 index 0000000000000000000000000000000000000000..a52bf1146c1b995cfcde3ed393487198c3771df7 Binary files /dev/null and b/deer.jpg differ diff --git a/egg.jpg b/egg.jpg new file mode 100644 index 0000000000000000000000000000000000000000..bb8a3e81c5ee2d2fb845785e68c5cb689a655ba3 Binary files /dev/null and b/egg.jpg differ diff --git a/fish.jpg b/fish.jpg new file mode 100644 index 0000000000000000000000000000000000000000..0196080967488e21e1e15934aaea45d75c1272f5 Binary files /dev/null and b/fish.jpg differ diff --git a/flagged/Input Image/88e8b1b238c064836037/HVITa2016a_000011.JPG b/flagged/Input Image/88e8b1b238c064836037/HVITa2016a_000011.JPG new file mode 100644 index 0000000000000000000000000000000000000000..6975666d7dd574cdcd519601b4460f015353e632 Binary files /dev/null and b/flagged/Input Image/88e8b1b238c064836037/HVITa2016a_000011.JPG differ diff --git a/flagged/log.csv b/flagged/log.csv new file mode 100644 index 0000000000000000000000000000000000000000..6ed3c371ec35a76d2dcdf0ecc6e2534d0a113571 --- /dev/null +++ b/flagged/log.csv @@ -0,0 +1,215 @@ +Input Image,Specify object to count here:,output,flag,username,timestamp +flagged/Input Image/88e8b1b238c064836037/HVITa2016a_000011.JPG,bird,flagged/output/a686c9813d1a6c436907/image.webp,,,2024-06-09 09:33:54.235722 +flagged/Input Image/875b25d454777d421c42/221113_ECCC_MRADUL_ELEPHANT_MIA_RHEA_EVENING_WALK_KEEPER-8-1920x900.jpg,people and elephants,,,flagged/Detected Instances/76fafa9d17afaaf9eb38/image.webp,4,,,2024-06-18 16:54:16.644520 +flagged/Input Image/037d1ea9a55366ad2940/2.jpg,shell,,"{""image"": ""flagged/Specify object to count with visual exemplars here/fceed5d5292244ecdd73/2.jpg"", ""points"": [[380.0, 211.0, 2.0, 483.0, 317.0, 3.0]]}",flagged/Detected Instances/27c94b87a5ced00b9dd1/image.webp,8,,,2024-06-22 08:03:07.969202 +flagged/Input Image/b8faa04105f66c5a71d1/7.jpg,woman,,,flagged/Detected Instances/cd67c449c0a10a91f35b/image.webp,1,,,2024-06-22 08:08:57.657953 +flagged/Input Image/350e00785dada901c520/7.jpg,woman,,,flagged/Detected Instances/b16f6a58d39f8b72ca49/image.webp,1,,,2024-06-22 08:18:43.429051 +flagged/Input Image/12d92058a4ac08b10dea/7.jpg,woman,,"{""image"": ""flagged/Specify object to count with visual exemplars here/38245c2c066faff3cae8/7.jpg"", ""points"": [[113.0, 132.0, 2.0, 260.0, 315.0, 3.0]]}",flagged/Detected Instances/b941dacc2ffe391d2e16/image.webp,10,,,2024-06-22 08:19:03.379704 +flagged/Input Image/9b95bcefbe76b48d43ea/39.jpg,,,"{""image"": ""flagged/Specify object to count with visual exemplars here/f7cb8955646797bfe577/39.jpg"", ""points"": [[218.0, 225.0, 2.0, 295.0, 282.0, 3.0]]}",flagged/Detected Instances/f1930c50e7cf23e1bab8/image.webp,20,,,2024-06-22 08:19:37.258705 +flagged/Input Image/a29aa3c68b71af80ceac/39.jpg,tomato,,"{""image"": ""flagged/Specify object to count with visual exemplars here/80430ff258140b26bd0e/39.jpg"", ""points"": [[218.0, 225.0, 2.0, 295.0, 282.0, 3.0]]}",flagged/Detected Instances/fc4d452719e25259f01d/image.webp,19,,,2024-06-22 08:19:58.499357 +flagged/Input Image/bdf516de7a637158f274/HVITa2016a_000011.JPG,bird,,,flagged/Detected Instances/d0d2e5d0a4a0ac61ac4d/image.webp,30,,,2024-06-22 09:39:09.374430 +flagged/Input Image/9d6b2710e64f8889dd64/HVITa2016a_000011.JPG,,,"{""image"": ""flagged/Specify object to count with visual exemplars here/453f32945cd9e681b483/KAPWa2019a_000132.JPG"", ""points"": [[510.0, 150.0, 2.0, 534.0, 176.0, 3.0], [684.0, 205.0, 2.0, 712.0, 224.0, 3.0]]}",flagged/Detected Instances/5c1af4b6118751ff9b9d/image.webp,32,,,2024-06-22 09:40:38.259957 +flagged/Input Image/5ca9673ad1540cff6509/HVITa2016a_000011.JPG,bird,,"{""image"": ""flagged/Specify object to count with visual exemplars here/5df88f87ed6641277db1/KAPWa2019a_000132.JPG"", ""points"": [[744.0, 429.0, 2.0, 763.0, 461.0, 3.0]]}",flagged/Detected Instances/f44fe32f6932dd6574b3/image.webp,31,,,2024-06-22 09:41:55.748604 +flagged/Input Image/b6a179bab6c6189ddad7/55.jpg,tomato,,,flagged/Detected Instances/6d315017249048f38b52/image.webp,50,,,2024-06-22 10:41:12.653937 +flagged/Input Image/9bf0e79d190639880832/55.jpg,tomato,,"{""image"": ""flagged/Specify object to count with visual exemplars here/fe802eb93b73f4dae6b7/42.jpg"", ""points"": [[180.0, 128.0, 2.0, 224.0, 162.0, 3.0]]}",flagged/Detected Instances/f1b139f485f3c6049bb5/image.webp,50,,,2024-06-22 10:41:37.183988 +flagged/Input Image/80d1ea6bfc5bc8002c62/28.jpg,cinnamon roll,,,flagged/Detected Instances/caac9eeb8dcadfd732cb/image.webp,8,,,2024-06-22 11:13:59.554789 +flagged/Input Image/3f37e6da6e272866e8de/3.jpg,hot air balloon,,,flagged/Detected Instances/24844df4b3df513a0c36/image.webp,11,,,2024-06-22 14:11:32.948564 +flagged/Input Image/5e7dc59cf0a41b9bc5fb/fish images.jpeg,fish,,,flagged/Detected Instances/a96993ece7e66065ba7d/image.webp,77,,,2024-06-22 14:16:37.842263 +flagged/Input Image/57a992296c0ec254b864/Fish 2.jpeg,fish,,,flagged/Detected Instances/2c9d451701856fe3371d/image.webp,900,,,2024-06-22 14:20:50.590216 +flagged/Input Image/6c3c11094015679fa790/Fish 3.jpeg,fish,,,flagged/Detected Instances/4e75007f5ef68e4ca3d6/image.webp,839,,,2024-06-22 14:22:09.322830 +flagged/Input Image/31698b5d2863b6756bfa/Fish 4.jpeg,fish,,,flagged/Detected Instances/9d32c5fb797b4985cee0/image.webp,278,,,2024-06-22 14:25:47.760426 +flagged/Input Image/5ce8e5a0fa4008aec1f2/Fish 4.jpeg,fish,,"{""image"": ""flagged/Specify object to count with visual exemplars here/6780b6a4f82e7d10d5cf/fish images.jpeg"", ""points"": [[63.0, 65.0, 2.0, 478.0, 217.0, 3.0]]}",flagged/Detected Instances/c6b81dedf92ab0af4153/image.webp,286,,,2024-06-22 14:28:36.854956 +flagged/Input Image/00be0bde65646930f991/Fish 4.jpeg,fish,,"{""image"": ""flagged/Specify object to count with visual exemplars here/25a3e4831f706db4a521/Fish 3.jpeg"", ""points"": [[319.0, 221.0, 2.0, 650.0, 516.0, 3.0]]}",flagged/Detected Instances/25d123e67ec8644c5f02/image.webp,277,,,2024-06-22 14:29:09.194220 +flagged/Input Image/542e2646fed421a4d5e5/6f91f056-e447-4316-aae1-b1ca57b55ec6.jpeg,Man,,,flagged/Detected Instances/4e890d293a4cc098b2b5/image.webp,1,,,2024-06-22 14:29:55.868319 +flagged/Input Image/5571766c4ae584f798f6/Fish 3.jpeg,fish and fish with weird and large mouth,,,flagged/Detected Instances/9f53a1f8ee034afcb29f/image.webp,347,,,2024-06-22 14:32:29.009512 +flagged/Input Image/df8f6e4f3c2658165b32/Fish 3.jpeg,fish and fish with weird and large mouth,,,flagged/Detected Instances/9c453d75fc3f0b7ba779/image.webp,347,,,2024-06-22 14:33:38.795075 +flagged/Input Image/52226488b0c44d964af9/Fish 3.jpeg,All fishes big and small,,,flagged/Detected Instances/bf38e737886a0ac61746/image.webp,609,,,2024-06-22 14:34:34.754255 +flagged/Input Image/caae84890c10f63e5c53/Fish 3.jpeg,All fish big and small,,,flagged/Detected Instances/d209f5a62318a3cc9a83/image.webp,653,,,2024-06-22 14:34:49.612537 +flagged/Input Image/b1986e95b8a8df57d81b/Fish 3.jpeg,A big fish and all small fish,,,flagged/Detected Instances/10fdbef3f9c29b00d343/image.webp,655,,,2024-06-22 14:36:39.308451 +flagged/Input Image/b320c79d24f53c48cb36/Fish 3.jpeg,A big fish and all small fishes,,,flagged/Detected Instances/83948fd3cdfc64acba91/image.webp,663,,,2024-06-22 14:37:47.440375 +flagged/Input Image/abf54b47c8c97ae29e2d/Fish 3.jpeg,,,"{""image"": ""flagged/Specify object to count with visual exemplars here/0b10284a52f55d2b997c/Fish 3.jpeg"", ""points"": [[738.0, 295.0, 2.0, 917.0, 469.0, 3.0]]}",flagged/Detected Instances/2bb2c04de9c3c15284a2/image.webp,595,,,2024-06-22 14:42:47.880099 +flagged/Input Image/485ab0f7f21cef342c33/Fish 3.jpeg,,,"{""image"": ""flagged/Specify object to count with visual exemplars here/a2645b60d75a35644c30/Fish 3.jpeg"", ""points"": [[717.0, 65.0, 2.0, 934.0, 232.0, 3.0], [812.0, 55.0, 2.0, 812.0, 155.0, 3.0]]}",flagged/Detected Instances/bb94b03bbae3744cb687/image.webp,710,,,2024-06-22 14:43:57.785652 +flagged/Input Image/eaf2d446d413a9b288cf/Fish 3.jpeg,A big fish and all small fishes,,"{""image"": ""flagged/Specify object to count with visual exemplars here/8848ccbc7b6f9793d87a/Fish 3.jpeg"", ""points"": [[717.0, 65.0, 2.0, 934.0, 232.0, 3.0], [812.0, 55.0, 2.0, 812.0, 155.0, 3.0]]}",flagged/Detected Instances/31643c3688da9b4fa531/image.webp,716,,,2024-06-22 14:44:56.566214 +flagged/Input Image/432fc52db57386081a7f/Fish 3.jpeg,A big fish and all small fishes,,"{""image"": ""flagged/Specify object to count with visual exemplars here/b3b4cf1ebceac265a27a/Fish 3.jpeg"", ""points"": [[202.0, 10.0, 2.0, 678.0, 280.0, 3.0]]}",flagged/Detected Instances/596768cb96359baae656/image.webp,736,,,2024-06-22 14:46:10.694967 +flagged/Input Image/a12f368ae7c4c5fd3da6/Fish 3.jpeg,,,"{""image"": ""flagged/Specify object to count with visual exemplars here/a59d14a310592e735186/Fish 3.jpeg"", ""points"": [[44.0, 96.0, 2.0, 83.0, 148.0, 3.0]]}",flagged/Detected Instances/663747d5d817f3b6f17b/image.webp,603,,,2024-06-22 14:50:04.561496 +flagged/Input Image/e081cd4b3f8c0a233e93/HVITa2016a_000011.JPG,bird,,,flagged/Detected Instances/ef9c94eaef8c71a59be7/image.webp,30,,,2024-06-22 14:59:44.195345 +flagged/Input Image/037b6400419594ee083f/2024-06-20 21.21.07.jpg,apples,,,flagged/Detected Instances/69304bef5b817c19a40d/image.webp,0,,,2024-06-22 21:16:37.304942 +flagged/Input Image/7fb25053e99ea97dec5d/IMG_4345.jpeg,bottle of water,,,flagged/Detected Instances/cd9ef90fd76f3c5bc28d/image.webp,5,,,2024-06-22 21:18:02.687111 +flagged/Input Image/ac22a60546af5b9e27d5/Julie_Chandelier_South-Georgia_Gold-Harbour_King-Penguins-colony-2-scaled.jpg,,,"{""image"": ""flagged/Specify object to count with visual exemplars here/9df284bf10e2e549b58b/900.jpeg"", ""points"": [[254.0, 43.0, 2.0, 615.0, 646.0, 3.0]]}",flagged/Detected Instances/6afde69e615172163cd9/image.webp,19,,,2024-06-22 21:24:24.843762 +flagged/Input Image/1e3fbaf70a0e480fcc7e/photo-1536842717890-4804617eb992.jpeg,balloon,,,flagged/Detected Instances/9e35897bb2b104feaf01/image.webp,21,,,2024-06-23 18:19:01.914350 +flagged/Input Image/8c8253a9745752f7ac5c/photo-1533551131698-c87deddfa2d6.jpeg,person,,,flagged/Detected Instances/f73654df476be2e2025a/image.webp,392,,,2024-06-23 18:20:38.923697 +flagged/Input Image/ba84aaaf18399d9d7285/photo-1533551131698-c87deddfa2d6.jpeg,person,,"{""image"": ""flagged/Specify object to count with visual exemplars here/2887c60fea7795aacecb/photo-1533551131698-c87deddfa2d6.jpeg"", ""points"": [[471.0, 235.0, 2.0, 529.0, 335.0, 3.0]]}",flagged/Detected Instances/5128711a7e3b3b296705/image.webp,379,,,2024-06-23 18:21:23.083042 +flagged/Input Image/288c3bba8e73832aa8a2/photo-1533551131698-c87deddfa2d6.jpeg,,,"{""image"": ""flagged/Specify object to count with visual exemplars here/8af1f367a37b6758f689/photo-1533551131698-c87deddfa2d6.jpeg"", ""points"": [[471.0, 235.0, 2.0, 529.0, 335.0, 3.0]]}",flagged/Detected Instances/26a1963fd9197c1e4c1d/image.webp,312,,,2024-06-23 18:21:38.559234 +flagged/Input Image/254816dc70cdd88ffa6c/photo-1533551131698-c87deddfa2d6.jpeg,person,,,flagged/Detected Instances/6bb3d26c43dc6b20eeba/image.webp,392,,,2024-06-23 18:21:50.311989 +flagged/Input Image/25d39fcb0445819504c4/photo-1533551131698-c87deddfa2d6.jpeg,person,,"{""image"": ""flagged/Specify object to count with visual exemplars here/005d56113a6597d456d6/photo-1533551131698-c87deddfa2d6.jpeg"", ""points"": [[473.0, 242.0, 2.0, 526.0, 339.0, 3.0]]}",flagged/Detected Instances/ef8e0debab490fee3637/image.webp,381,,,2024-06-23 18:22:20.986792 +flagged/Input Image/2bc6a37e212dcc397a14/JUTeHjN.jpg,,,"{""image"": ""flagged/Specify object to count with visual exemplars here/1dfc2902049a063b4d3b/JUTeHjN.jpg"", ""points"": [[274.0, 559.0, 2.0, 301.0, 639.0, 3.0]]}",flagged/Detected Instances/d0472660028ee835a909/image.webp,591,,,2024-06-23 18:26:11.912055 +flagged/Input Image/cc6b05c49c410ce714a4/JUTeHjN.jpg,waldo,,"{""image"": ""flagged/Specify object to count with visual exemplars here/0354019aa33aa8cc1b4c/JUTeHjN.jpg"", ""points"": [[274.0, 559.0, 2.0, 301.0, 639.0, 3.0]]}",flagged/Detected Instances/e7d80138cd030c1a4de7/image.webp,707,,,2024-06-23 18:26:56.983776 +flagged/Input Image/c1f5e295668daa58b78b/La_horde_-_Surfers_riding_a_wave_in_Paea_Tahiti.jpg,people,,,flagged/Detected Instances/2b8161f3e7ac96bc81a6/image.webp,6,,,2024-06-24 08:54:20.028204 +flagged/Input Image/3077b7ba647265981dac/Screenshot from 2024-06-18 07-24-03.png,letter t,,,flagged/Detected Instances/d0f7006b69b870c7c227/image.webp,60,,,2024-06-24 13:39:18.019532 +flagged/Input Image/9a11e4fe8738ffdc2176/Screenshot from 2024-06-18 07-24-03.png,letter t,,"{""image"": ""flagged/Specify object to count with visual exemplars here/441e53d9531d94725262/hello.png"", ""points"": []}",flagged/Detected Instances/0465e358fdb89e04a940/image.webp,60,,,2024-06-24 13:40:57.783042 +flagged/Input Image/afa9d75359a32a6d4b41/Screenshot from 2024-06-18 07-24-03.png,letter t,,"{""image"": ""flagged/Specify object to count with visual exemplars here/037cdd9bcd4683726fc5/hello.png"", ""points"": []}",flagged/Detected Instances/2f592103e85d7e6e527a/image.webp,60,,,2024-06-24 13:41:29.438433 +flagged/Input Image/57809c4a9bafbbd30918/Screenshot from 2024-06-18 07-24-03.png,Ts,,"{""image"": ""flagged/Specify object to count with visual exemplars here/e22fa571644aa160d86c/hello.png"", ""points"": []}",flagged/Detected Instances/c318b58b8c88b7710e0c/image.webp,6,,,2024-06-24 13:41:45.916615 +flagged/Input Image/c9412ee3b51eacd33f52/06.jpg_instances.jpg,fish,,,flagged/Detected Instances/a6c2ad6ab9379e0211fa/image.webp,9,,,2024-06-24 15:07:10.854678 +flagged/Input Image/e2976e4738abc316313f/word-image-233.jpeg,leaves,,,flagged/Detected Instances/e8eab3eeef0d444bbb1a/image.webp,10,,,2024-06-24 15:53:21.817485 +flagged/Input Image/be3ab6fa387c0a0194f4/word-image-233.jpeg,leaf,,,flagged/Detected Instances/90dc5680d2ec80c0eac8/image.webp,10,,,2024-06-24 15:55:43.676702 +flagged/Input Image/8ee3a4ce6a33884d03ae/2.jpg,sea shell,,,flagged/Detected Instances/f0d8fd3020ac899200f2/image.webp,8,,,2024-06-24 15:57:17.391770 +flagged/Input Image/74980bc345053ad14f89/215.jpg,grape,,,flagged/Detected Instances/e73eae638535e8770764/image.webp,243,,,2024-06-24 15:58:37.852766 +flagged/Input Image/d206f8d836baf220ac41/HVITa2016a_000011.JPG,bird,,,flagged/Detected Instances/9949e21c2f5bacf777a1/image.webp,30,,,2024-06-24 16:27:43.463142 +flagged/Input Image/86a081dc917ef9d837c7/HVITa2016a_000011.JPG,,,"{""image"": ""flagged/Specify object to count with visual exemplars here/64abdac51a65819724a6/HVITa2016a_000011.JPG"", ""points"": [[1230.0, 118.0, 2.0, 1292.0, 201.0, 3.0], [830.0, 550.0, 2.0, 906.0, 623.0, 3.0]]}",flagged/Detected Instances/d92c3d3991cfcefde001/image.webp,33,,,2024-06-24 16:28:22.627606 +flagged/Input Image/35151af517a911bcc223/HVITa2016a_000011.JPG,bird,,"{""image"": ""flagged/Specify object to count with visual exemplars here/4f611be7fe99047414ae/HVITa2016a_000011.JPG"", ""points"": [[1230.0, 118.0, 2.0, 1292.0, 201.0, 3.0], [830.0, 550.0, 2.0, 906.0, 623.0, 3.0]]}",flagged/Detected Instances/10125734c7afa38ed087/image.webp,31,,,2024-06-24 16:28:36.193649 +flagged/Input Image/bba1422fdfed279a73af/HVITa2016a_000011.JPG,bird,,flagged/Detected Instances/bec28a276df78afe343d/image.webp,30,,,2024-06-24 16:41:23.500662 +flagged/Input Image/d7762005e6674cf12e5c/HVITa2016a_000011.JPG,bird,,flagged/Detected Instances/cca9e646a051b17f95f5/image.webp,30,,,2024-06-24 17:55:54.474905 +flagged/Input Image/81aa794f290cedf734c3/HVITa2016a_000011.JPG,,"{""image"": ""flagged/Specify object to count with visual exemplars here/6da159a3958dedc078b2/HVITa2016a_000011.JPG"", ""points"": []}",flagged/Detected Instances/02faafb810646f2ea9b4/image.webp,0,,,2024-06-24 17:56:45.209436 +flagged/Input Image/7b8f501cc28c308ff8b6/HVITa2016a_000011.JPG,,"{""image"": ""flagged/Specify object to count with visual exemplars here/4a30c21797e2739e4abc/HVITa2016a_000011.JPG"", ""points"": [[943.0, 374.0, 2.0, 1057.0, 485.0, 3.0]]}",flagged/Detected Instances/e27a207c035db1ab701e/image.webp,32,,,2024-06-24 18:03:25.307742 +flagged/Input Image/7af486902d9e28508494/HVITa2016a_000011.JPG,,"{""image"": ""flagged/Specify object to count with visual exemplars here/4a1a53289daf6c4ab771/HVITa2016a_000011.JPG"", ""points"": []}",flagged/Detected Instances/2d8b8e39fc9512762815/image.webp,0,,,2024-06-24 18:43:48.203081 +flagged/Input Image/55afacf49af9b719cdda/HVITa2016a_000011.JPG,bird,"{""image"": ""flagged/Specify object to count with visual exemplars here/30839117a5e1c2687952/HVITa2016a_000011.JPG"", ""points"": [[1230.0, 118.0, 2.0, 1292.0, 201.0, 3.0], [830.0, 550.0, 2.0, 906.0, 623.0, 3.0]]}",flagged/Detected Instances/c9d49f2d15a7de44e80d/image.webp,31,,,2024-06-24 18:52:25.941469 +flagged/Input Image/266f273a429e2a835b60/HVITa2016a_000011.JPG,,"{""image"": ""flagged/Specify object to count with visual exemplars here/ea17095cd854670fd8db/HVITa2016a_000011.JPG"", ""points"": []}",flagged/Detected Instances/7f02a76d21f7e077559e/image.webp,0,,,2024-06-25 08:53:21.768814 +flagged/Input Image/e93ffc35e1c7c113beec/HVITa2016a_000011.JPG,,"{""image"": ""flagged/Specify object to count with visual exemplars here/af5231d7685032d2c2fe/HVITa2016a_000011.JPG"", ""points"": []}",flagged/Detected Instances/5cd4eb530194572ba703/image.webp,0,,,2024-06-25 09:16:55.226689 +flagged/Input Image/733b82dbcccd8aae5c53/HVITa2016a_000011.JPG,,"{""image"": ""flagged/Specify object to count with visual exemplars here/be9a0871ad384d450dbc/HVITa2016a_000011.JPG"", ""points"": []}",flagged/Detected Instances/d9e693ae5adb9314a442/image.webp,0,,,2024-06-25 09:22:48.487486 +flagged/Input Image/180e288224fa47f7e40e/HVITa2016a_000011.JPG,,"{""image"": ""flagged/Specify object to count with visual exemplars here/3b47f1cbc62a52b89a35/HVITa2016a_000011.JPG"", ""points"": []}",flagged/Detected Instances/c06ad461ce25c9e93519/image.webp,0,,,2024-06-25 09:37:10.918912 +flagged/Input Image/f6c46243c9e41736a91b/HVITa2016a_000011.JPG,bird,,flagged/Detected Instances/f834fe8aa737c6602250/image.webp,30,,,2024-06-25 09:40:18.398469 +flagged/Input Image/02e6dd294378714d2c60/HVITa2016a_000011.JPG,,"{""image"": ""flagged/Specify object to count with visual exemplars here/7a28c64f2d7f386ba90f/HVITa2016a_000011.JPG"", ""points"": []}",flagged/Detected Instances/58939acce9372835d94f/image.webp,0,,,2024-06-25 09:40:26.315283 +flagged/Input Image/ef125c4204764412c3f7/HVITa2016a_000011.JPG,,"{""image"": ""flagged/Specify object to count with visual exemplars here/5130a669c02f296bb278/HVITa2016a_000011.JPG"", ""points"": []}",flagged/Detected Instances/befab1125336c5e50181/image.webp,0,,,2024-06-25 10:05:42.679549 +flagged/Input Image/feae1a999efc1b161e4a/HVITa2016a_000011.JPG,,"{""image"": ""flagged/Specify object to count with visual exemplars here/d8f9a71db6b70a468a0f/HVITa2016a_000011.JPG"", ""points"": []}",flagged/Detected Instances/ec35f973d14b7fc08792/image.webp,0,,,2024-06-25 10:14:45.281246 +flagged/Input Image/b721bed7a52d56eabff8/HVITa2016a_000011.JPG,,"{""image"": ""flagged/Specify object to count with visual exemplars here/8df3b7b4105c8bbed352/HVITa2016a_000011.JPG"", ""points"": [[1224.0, 99.0, 2.0, 1308.0, 215.0, 3.0]]}",flagged/Detected Instances/1ab68fe2b72b062794c3/image.webp,32,,,2024-06-25 10:15:02.226475 +flagged/Input Image/30c0dd916e13e8cdf40a/HVITa2016a_000011.JPG,,"{""image"": ""flagged/Specify object to count with visual exemplars here/3183ccd48e9876f48116/HVITa2016a_000011.JPG"", ""points"": []}",flagged/Detected Instances/486ddeee601a5658d3c1/image.webp,0,,,2024-06-25 10:17:04.340172 +flagged/Input Image/59fcdf686244d693b758/HVITa2016a_000011.JPG,bird,,flagged/Detected Instances/1c4afcc87d7b113064e0/image.webp,30,,,2024-06-25 10:25:55.780204 +flagged/Input Image/0faa5dfd6e852c50d8ff/HVITa2016a_000011.JPG,,"{""image"": ""flagged/Specify object to count with visual exemplars here/5e3aabee37bf681d51a4/HVITa2016a_000011.JPG"", ""points"": []}",flagged/Detected Instances/2cd238061edc54130cdf/image.webp,0,,,2024-06-25 10:26:05.084390 +flagged/Input Image/3e839a4e8ebe95002a20/HVITa2016a_000011.JPG,bird,"{""image"": ""flagged/Specify object to count with visual exemplars here/f0cd0be5ea8ccbb26d0f/HVITa2016a_000011.JPG"", ""points"": []}",flagged/Detected Instances/b8c1235f9f45df306ef9/image.webp,30,,,2024-06-25 10:26:13.747497 +flagged/Input Image/aea44fcfb41612489dad/HVITa2016a_000011.JPG,,"{""image"": ""flagged/Specify object to count with visual exemplars here/da461ef7d215c55a5e7c/HVITa2016a_000011.JPG"", ""points"": []}",flagged/Detected Instances/cf0299a8a5c332b5c02f/image.webp,0,,,2024-06-25 10:27:17.764060 +flagged/Input Image/defc48612eca2926a7b7/bird-1.JPG,bird,"{""image"": ""flagged/Specify object to count with visual exemplars here/bf6d9e9b6fd1b4962765/bird-2.JPG"", ""points"": []}",flagged/Detected Instances/8a857874d6e33125d752/image.webp,30,,,2024-06-25 10:56:25.591931 +flagged/Input Image/02215a04cfb823dd7796/bird-1.JPG,bird,"{""image"": ""flagged/Specify object to count with visual exemplars here/54a44757b38db813d618/bird-2.JPG"", ""points"": [[608.0, 232.0, 2.0, 640.0, 263.0, 3.0]]}",flagged/Detected Instances/c0ed66b9cad56b199aa2/image.webp,31,,,2024-06-25 10:59:54.183533 +flagged/Input Image/df6cea1edf80683a3767/bird-1.JPG,bird,"{""image"": ""flagged/CLICK DRAG to specify object to count with visual exemplars here/0dce00cba984dd532757/bird-2.JPG"", ""points"": [[503.0, 142.0, 2.0, 542.0, 184.0, 3.0]]}",flagged/Detected Instances/c1f230dd1e883d9e32e7/image.webp,31,,,2024-06-25 11:15:22.282716 +flagged/Input Image/8d0fb36ef83e86fd525a/strawberry.jpg,strawberry,"{""image"": ""flagged/CLICK DRAG to specify object to count with visual exemplars here/e646d85a66c06474d22d/strawberry.jpg"", ""points"": [[160.0, 68.0, 2.0, 197.0, 138.0, 3.0]]}",flagged/Detected Instances/ed9419605b016fba2dc5/image.webp,16,,,2024-06-25 11:30:30.236817 +flagged/Input Image/b49396917cf5ecc697fa/strawberry.jpg,blueberry,"{""image"": ""flagged/CLICK DRAG to specify object to count with visual exemplars here/f3fd414caafcebcbda4a/strawberry.jpg"", ""points"": []}",flagged/Detected Instances/8632f9c1bb05eee15dcc/image.webp,137,,,2024-06-25 11:30:43.620161 +flagged/Input Image/3dd1700ef3bcbe5657f7/strawberry.jpg,blueberry,"{""image"": ""flagged/CLICK DRAG to specify object to count with visual exemplars here/5dfc0e9e8efa0af0c6a8/strawberry.jpg"", ""points"": [[253.0, 130.0, 2.0, 273.0, 152.0, 3.0]]}",flagged/Detected Instances/94ce91cd91c43e18974d/image.webp,133,,,2024-06-25 11:30:53.267942 +flagged/Input Image/cec4554c9b13aa4d346d/clipboard.png,fish,,flagged/Detected Instances/1830def027aa0d3f3d91/image.webp,434,,,2024-06-25 11:38:27.652346 +flagged/Input Image/032c2e1714788159e262/clipboard.png,fish,"{""image"": ""flagged/CLICK DRAG to specify object to count with visual exemplars here/04a1a12a302c9c1db814/clipboard.png"", ""points"": [[1049.0, 213.0, 2.0, 1125.0, 238.0, 3.0]]}",flagged/Detected Instances/122ff6e9f6ff79c3e5a7/image.webp,411,,,2024-06-25 11:39:25.658141 +flagged/Input Image/5422ce8f8f4364819aaa/clipboard.png,fish,"{""image"": ""flagged/CLICK DRAG to specify object to count with visual exemplars here/c32361eb60578a2bd4ac/clipboard.png"", ""points"": []}",flagged/Detected Instances/5d5df763986fec98e93f/image.webp,900,,,2024-06-25 11:41:22.135329 +flagged/Input Image/5072a86417c33eb593f3/clipboard.png,fish,"{""image"": ""flagged/CLICK DRAG to specify object to count with visual exemplars here/028b7ae649c4dd8b7178/clipboard.png"", ""points"": [[1832.0, 158.0, 2.0, 1878.0, 223.0, 3.0]]}",flagged/Detected Instances/c65818f6571e4d168e43/image.webp,900,,,2024-06-25 11:41:46.118310 +flagged/Input Image/df7f9cdf52e79013f8cb/clipboard.png,fish,"{""image"": ""flagged/CLICK DRAG to specify object to count with visual exemplars here/09eb9b77eff2e836b6ff/clipboard.png"", ""points"": []}",flagged/Detected Instances/b4a6138c92f1c746ccdb/image.webp,248,,,2024-06-25 11:42:43.425077 +flagged/Input Image/a9ccd350937abcb63291/clipboard.png,fish,"{""image"": ""flagged/CLICK DRAG to specify object to count with visual exemplars here/19ce574f4470218dc7e0/clipboard.png"", ""points"": [[367.0, 183.0, 2.0, 451.0, 219.0, 3.0]]}",flagged/Detected Instances/d48cc5c8133c9068a0dd/image.webp,247,,,2024-06-25 11:43:33.822691 +flagged/Input Image/a3d2036f9c1ac12e961c/fish.jpg,fish,"{""image"": ""flagged/CLICK DRAG to specify object to count with visual exemplars here/0c6da89b80bb001aa49e/fish.jpg"", ""points"": []}",flagged/Detected Instances/f27dfe196a1c41a69503/image.webp,248,,,2024-06-25 11:45:59.667866 +flagged/Input Image/46f4410a1aa98f5ee2af/strawberry.jpg,blueberry,"{""image"": ""flagged/CLICK DRAG to specify object to count with visual exemplars here/2fd7f3ce369404120141/strawberry.jpg"", ""points"": []}",flagged/Detected Instances/0197e9eeac0554007c3f/image.webp,137,,,2024-06-25 11:46:32.895500 +flagged/Input Image/bb268ffa2de20ebe1c26/strawberry.jpg,blueberry,"{""image"": ""flagged/CLICK DRAG to specify object to count with visual exemplars here/c3b2d652d852a7eab171/strawberry.jpg"", ""points"": [[342.0, 126.0, 2.0, 359.0, 144.0, 3.0]]}",flagged/Detected Instances/44f0249ecddfb4155756/image.webp,137,,,2024-06-25 11:46:42.177472 +flagged/Input Image/6c66434da1d48433ecb3/strawberry.jpg,blueberry,"{""image"": ""flagged/CLICK DRAG to specify object to count with visual exemplars here/4d4cffebe4515feff0ec/strawberry.jpg"", ""points"": []}",flagged/Detected Instances/ab963c869b4f8d820e49/image.webp,137,,,2024-06-25 11:46:57.009420 +flagged/Input Image/a6875dccb70b93acd740/strawberry.jpg,blueberry,"{""image"": ""flagged/CLICK DRAG to specify object to count with visual exemplars here/f481446bb34bcb55993f/strawberry.jpg"", ""points"": [[333.0, 167.0, 2.0, 353.0, 187.0, 3.0], [301.0, 100.0, 2.0, 321.0, 119.0, 3.0]]}",flagged/Detected Instances/bdf5762c87a5fb86e0b2/image.webp,137,,,2024-06-25 11:47:18.226311 +flagged/Input Image/44cef4325bfc0b2aa091/strawberry.jpg,strawberry,"{""image"": ""flagged/CLICK DRAG to specify object to count with visual exemplars here/bcaa94ade9f2b6274f1c/strawberry.jpg"", ""points"": []}",flagged/Detected Instances/613e0b5dca0804e918c7/image.webp,16,,,2024-06-25 11:47:34.461816 +flagged/Input Image/6d93ce821c043cc1fa50/strawberry.jpg,blueberry,"{""image"": ""flagged/CLICK DRAG to specify object to count with visual exemplars here/ae226a70b29c89cc4eb8/strawberry.jpg"", ""points"": []}",flagged/Detected Instances/00176ba3de1e4fbd2c79/image.webp,137,,,2024-06-25 11:47:45.327559 +flagged/Input Image/214729be550d569c1c68/bird-1.JPG,bird,"{""image"": ""flagged/CLICK DRAG to specify object to count with visual exemplars here/f6eed5c264264d847ff6/bird-2.JPG"", ""points"": []}",flagged/Detected Instances/c12f210c5902dfd03e70/image.webp,30,,,2024-06-25 11:48:34.856890 +flagged/Input Image/8bf133a0564fd92cc890/balloon.jpg,hot air balloon,"{""image"": ""flagged/CLICK DRAG to specify object to count with visual exemplars here/978150756fa4f1ef5027/balloon.jpg"", ""points"": []}",flagged/Detected Instances/5b47d64211343b63152a/image.webp,108,,,2024-06-25 12:25:57.589269 +flagged/Input Image/6429656af3e526d1c231/women.jpg,women,"{""image"": ""flagged/CLICK DRAG to specify object to count with visual exemplars here/25350112732936ba546b/women.jpg"", ""points"": []}",flagged/Detected Instances/be57da4ba536ae483835/image.webp,5,,,2024-06-25 12:26:41.171433 +flagged/Input Image/c58a77f0b28ca5d81b43/women.jpg,men,"{""image"": ""flagged/CLICK DRAG to specify object to count with visual exemplars here/130e25a655d338fdfe3e/women.jpg"", ""points"": []}",flagged/Detected Instances/e7e68bb358caa2a02fad/image.webp,6,,,2024-06-25 12:26:49.904389 +flagged/Input Image/a7f9b2e70fc2dd48ea61/women.jpg,man,"{""image"": ""flagged/CLICK DRAG to specify object to count with visual exemplars here/cd6645e130252cb0ce54/women.jpg"", ""points"": []}",flagged/Detected Instances/7ffc5ec4d2638a7d5993/image.webp,4,,,2024-06-25 12:26:59.917970 +flagged/Input Image/42851096a5465744aff8/women.jpg,boy,"{""image"": ""flagged/CLICK DRAG to specify object to count with visual exemplars here/fdaea5a42ce12dafba02/women.jpg"", ""points"": []}",flagged/Detected Instances/47cfc5281c09ccad9f3c/image.webp,1,,,2024-06-25 12:27:25.852703 +flagged/Input Image/67132ca41a6c48bc4ae8/women.jpg,girl,"{""image"": ""flagged/CLICK DRAG to specify object to count with visual exemplars here/2410b57b1dae0d986036/women.jpg"", ""points"": []}",flagged/Detected Instances/b8ef07f2a0d78188051d/image.webp,5,,,2024-06-25 12:27:35.919728 +flagged/Input Image/d66d3772b1409e1852d8/strawberry.jpg,strawberry,"{""image"": ""flagged/CLICK DRAG to specify object to count with visual exemplars here/2f7a42ec53940a767ab1/strawberry.jpg"", ""points"": []}",flagged/Detected Instances/db7ecc53cc8f29566958/image.webp,16,,,2024-06-25 12:29:35.376726 +flagged/Input Image/7493d467be9cfe1dff7c/strawberry.jpg,strawberry,"{""image"": ""flagged/CLICK DRAG to specify object to count with visual exemplars here/8cab0399346d70c00fc0/strawberry.jpg"", ""points"": [[338.0, 123.0, 2.0, 361.0, 146.0, 3.0]]}",flagged/Detected Instances/f01f6a02720e21ce0150/image.webp,50,,,2024-06-25 12:29:45.148685 +flagged/Input Image/440c504736080f8142ad/strawberry.jpg,blueberry,"{""image"": ""flagged/CLICK DRAG to specify object to count with visual exemplars here/92fd535788251da6ba09/strawberry.jpg"", ""points"": [[338.0, 123.0, 2.0, 361.0, 146.0, 3.0]]}",flagged/Detected Instances/990386fa1a9553c882db/image.webp,136,,,2024-06-25 12:29:57.059622 +flagged/Input Image/fed431554a261ab266a4/strawberry.jpg,blueberry,"{""image"": ""flagged/CLICK DRAG to specify object to count with visual exemplars here/60afc4b94764dc018370/strawberry.jpg"", ""points"": []}",flagged/Detected Instances/5f048bdb5003eb64e17b/image.webp,137,,,2024-06-25 12:30:08.506742 +flagged/Input Image/5e49dbc792a65c4dbbb5/strawberry.jpg,blueberry,"{""image"": ""flagged/CLICK DRAG to specify object to count with visual exemplars here/8415524ab37ceaf7469b/strawberry.jpg"", ""points"": [[338.0, 123.0, 2.0, 361.0, 146.0, 3.0], [257.0, 112.0, 2.0, 287.0, 136.0, 3.0]]}",flagged/Detected Instances/e56fa7694cc83977e48e/image.webp,137,,,2024-06-25 12:30:18.906628 +flagged/Input Image/daa4ac7838796afe20fd/bird-1.JPG,bird,"{""image"": ""flagged/CLICK DRAG to specify object to count with visual exemplars here/69dcd045d1a47c1a45cc/bird-2.JPG"", ""points"": []}",flagged/Detected Instances/7c51fda26eae4f4660a6/image.webp,30,,,2024-06-25 12:30:29.347865 +flagged/Input Image/7ba4ec104b5a3a146474/bird-1.JPG,bird,"{""image"": ""flagged/CLICK DRAG to specify object to count with visual exemplars here/164da9e8be790ea4ef79/bird-2.JPG"", ""points"": [[505.0, 147.0, 2.0, 536.0, 183.0, 3.0]]}",flagged/Detected Instances/f16c10ef2bc36b5dc69d/image.webp,31,,,2024-06-25 12:30:39.178460 +flagged/Input Image/420750e47befc70801e2/fish.jpg,fish,"{""image"": ""flagged/CLICK DRAG to specify object to count with visual exemplars here/9e22ac918784790aa880/fish.jpg"", ""points"": []}",flagged/Detected Instances/27ff780f6a23afc048b2/image.webp,248,,,2024-06-25 12:31:01.375887 +flagged/Input Image/1feeba99c7d2efe9096e/women.jpg,girl,"{""image"": ""flagged/CLICK DRAG to specify object to count with visual exemplars here/4f7337051dee8ff77653/women.jpg"", ""points"": []}",flagged/Detected Instances/ce0dd6ff795456176eab/image.webp,5,,,2024-06-25 12:31:17.643196 +flagged/Input Image/9f1c5da2733fa8c58ae8/women.jpg,boy,"{""image"": ""flagged/CLICK DRAG to specify object to count with visual exemplars here/d41a6c081af41e9a2f62/women.jpg"", ""points"": []}",flagged/Detected Instances/620e679c66d0f9574fad/image.webp,1,,,2024-06-25 12:31:28.806538 +flagged/Input Image/30598862ab850cd6db31/balloon.jpg,hot air balloon,"{""image"": ""flagged/CLICK DRAG to specify object to count with visual exemplars here/ff94594958fb2da48621/balloon.jpg"", ""points"": []}",flagged/Detected Instances/16993819652dc9bce0b8/image.webp,108,,,2024-06-25 12:31:44.635515 +flagged/Input Image/f9dbff9baf36cf6c8a91/balloon.jpg,hot air balloon,"{""image"": ""flagged/CLICK DRAG to specify object to count with visual exemplars here/39ac7da130be6fb0cba1/balloon.jpg"", ""points"": [[255.0, 38.0, 2.0, 274.0, 60.0, 3.0]]}",flagged/Detected Instances/fbaa66b107151c86b5f3/image.webp,108,,,2024-06-25 12:31:53.008190 +flagged/Input Image/86ef5e44d86d09969fa3/deer.jpg,deer,"{""image"": ""flagged/CLICK DRAG to specify object to count with visual exemplars here/fbc70e37fee443170ff4/deer.jpg"", ""points"": []}",flagged/Detected Instances/d1ab59ab9d6d21e0c4b3/image.webp,21,,,2024-06-25 12:32:16.020013 +flagged/Input Image/279428723858139b5a4c/apple.jpg,apple,"{""image"": ""flagged/CLICK DRAG to specify object to count with visual exemplars here/ea6e4f53ba1edf38def6/apple.jpg"", ""points"": []}",flagged/Detected Instances/f45dc48dfdc3f7df6249/image.webp,568,,,2024-06-25 12:32:45.752592 +flagged/Input Image/8213864ae0272f59994f/apple.jpg,apple,"{""image"": ""flagged/CLICK DRAG to specify object to count with visual exemplars here/d5d645dce3a0bedcf3c7/apple.jpg"", ""points"": [[316.0, 230.0, 2.0, 365.0, 273.0, 3.0]]}",flagged/Detected Instances/8f1288ccce0a4abba02c/image.webp,550,,,2024-06-25 12:33:07.817865 +flagged/Input Image/1482d3171ccab0738d15/egg.jpg,egg,"{""image"": ""flagged/CLICK DRAG to specify object to count with visual exemplars here/937d3cd3614befcd3692/egg.jpg"", ""points"": []}",flagged/Detected Instances/577b9c0ccd91d4d879ed/image.webp,18,,,2024-06-25 12:33:27.798499 +flagged/Input Image/25f50a2c8a3aa7d190bd/egg.jpg,egg,"{""image"": ""flagged/CLICK DRAG to specify object to count with visual exemplars here/d345ceb5293859fabf1c/egg.jpg"", ""points"": [[418.0, 85.0, 2.0, 472.0, 148.0, 3.0]]}",flagged/Detected Instances/87eb5f1e0f18dc6739f6/image.webp,18,,,2024-06-25 12:33:35.566274 +flagged/Input Image/9f0db0ff082f6b31f825/stamp.jpg,stamp,"{""image"": ""flagged/CLICK DRAG to specify object to count with visual exemplars here/701c11e28665c59565cc/stamp.jpg"", ""points"": []}",flagged/Detected Instances/82cabb88e68e8ecc63ba/image.webp,45,,,2024-06-25 12:33:47.051392 +flagged/Input Image/fcb41365c9ac544c8898/green-pea.jpg,green pea,"{""image"": ""flagged/CLICK DRAG to specify object to count with visual exemplars here/2b2c58e68049e4dade9d/green-pea.jpg"", ""points"": []}",flagged/Detected Instances/3f41f7d66283c8f516ec/image.webp,213,,,2024-06-25 12:33:59.684009 +flagged/Input Image/8a701be26fb9fb2d40e8/lego.jpg,lego,"{""image"": ""flagged/CLICK DRAG to specify object to count with visual exemplars here/87e17ecb546906d99293/lego.jpg"", ""points"": []}",flagged/Detected Instances/e1adec40dc3e0880d984/image.webp,23,,,2024-06-25 12:34:29.213218 +flagged/Input Image/4673ed3f053269ba11d4/lego.jpg,lego,"{""image"": ""flagged/CLICK DRAG to specify object to count with visual exemplars here/aae11dd32545f4f6e48a/lego.jpg"", ""points"": [[275.0, 166.0, 2.0, 294.0, 186.0, 3.0]]}",flagged/Detected Instances/0ed9da189663a3352717/image.webp,23,,,2024-06-25 12:34:41.379472 +flagged/Input Image/d956a7af5032db84ebf4/lego.jpg,lego,"{""image"": ""flagged/CLICK DRAG to specify object to count with visual exemplars here/65d7ec1fb0e4f2a6b378/lego.jpg"", ""points"": [[275.0, 166.0, 2.0, 294.0, 186.0, 3.0], [269.0, 256.0, 2.0, 284.0, 273.0, 3.0]]}",flagged/Detected Instances/f0675e3f8e5fbae79512/image.webp,23,,,2024-06-25 12:34:50.644433 +flagged/Input Image/4a629b96d8745c48c739/lego.jpg,lego,"{""image"": ""flagged/CLICK DRAG to specify object to count with visual exemplars here/a71c7ef0952943bac3d3/lego.jpg"", ""points"": [[275.0, 166.0, 2.0, 294.0, 186.0, 3.0], [269.0, 256.0, 2.0, 284.0, 273.0, 3.0], [124.0, 213.0, 2.0, 143.0, 232.0, 3.0]]}",flagged/Detected Instances/28d4f0037a145a77e53f/image.webp,23,,,2024-06-25 12:35:00.163591 +flagged/Input Image/be96be6a6471be11367b/fish.jpg,fish,"{""image"": ""flagged/CLICK DRAG to specify object to count with visual exemplars here/e94ed1719ff8f3280d11/fish.jpg"", ""points"": []}",flagged/Detected Instances/4ad9cd81c3a99b08a811/image.webp,248,,,2024-06-25 12:37:35.824495 +flagged/Input Image/5c18d75a43fc788202de/egg.jpg,egg,"{""image"": ""flagged/CLICK DRAG to specify object to count with visual exemplars here/1158914112f891b5efd0/egg.jpg"", ""points"": []}",flagged/Detected Instances/182a0d4d6453ae657719/image.webp,18,,,2024-06-25 12:38:00.758125 +flagged/Input Image/e0811fca1751db63b380/egg.jpg,,"{""image"": ""flagged/CLICK DRAG to specify object to count with visual exemplars here/382cdaec8615c7e2fecb/egg.jpg"", ""points"": [[199.0, 86.0, 2.0, 248.0, 142.0, 3.0]]}",flagged/Detected Instances/2079378c09fdd2a9c7ce/image.webp,18,,,2024-06-25 12:38:18.291809 +flagged/Input Image/d2e387b8a61c19868cba/apple.jpg,apple,"{""image"": ""flagged/CLICK DRAG to specify object to count with visual exemplars here/3ec031a562ec058e7731/apple.jpg"", ""points"": []}",flagged/Detected Instances/adf208e2ecead3e6a1f4/image.webp,568,,,2024-06-25 12:38:34.859201 +flagged/Input Image/5f63a040416264d96fc7/green-pea.jpg,green pea,"{""image"": ""flagged/CLICK DRAG to specify object to count with visual exemplars here/517ad35ce8c0e44b6782/green-pea.jpg"", ""points"": []}",flagged/Detected Instances/66a8dfe7bfefebd3121d/image.webp,213,,,2024-06-25 12:39:07.587414 +flagged/Input Image/aae210f02dbbc2bdd515/stamp.jpg,stamp,"{""image"": ""flagged/CLICK DRAG to specify object to count with visual exemplars here/422d122996b90782f0a3/stamp.jpg"", ""points"": []}",flagged/Detected Instances/6cbaa1c841d2eb3ec14d/image.webp,45,,,2024-06-25 12:42:10.960744 +flagged/Input Image/81719deb5b7c5097ebb0/stamp.jpg,square,"{""image"": ""flagged/CLICK DRAG to specify object to count with visual exemplars here/959e12c05af1e92d9355/stamp.jpg"", ""points"": []}",flagged/Detected Instances/21ac1219abab8e1a1e3a/image.webp,45,,,2024-06-25 12:42:30.712204 +flagged/Input Image/41061285b258efc14011/stamp.jpg,25,"{""image"": ""flagged/CLICK DRAG to specify object to count with visual exemplars here/b183dbda22012f6a2e91/stamp.jpg"", ""points"": []}",flagged/Detected Instances/e4fae1385b5fe0b62af2/image.webp,45,,,2024-06-25 12:42:48.778457 +flagged/Input Image/4c02c2917f07a0a47195/stamp.jpg,airplane,"{""image"": ""flagged/CLICK DRAG to specify object to count with visual exemplars here/f0ed657840030b178009/stamp.jpg"", ""points"": []}",flagged/Detected Instances/16c30614d826227da8a2/image.webp,0,,,2024-06-25 12:43:02.980277 +flagged/Input Image/965aa552db7559607920/stamp.jpg,horse,"{""image"": ""flagged/CLICK DRAG to specify object to count with visual exemplars here/bab35fa8d8c596d72c83/stamp.jpg"", ""points"": []}",flagged/Detected Instances/eb1276c9005a0e28ff80/image.webp,1,,,2024-06-25 12:43:22.910300 +flagged/Input Image/1f47a5922213f28e3d47/stamp.jpg,green,"{""image"": ""flagged/CLICK DRAG to specify object to count with visual exemplars here/cabeb714ed1e1cddefa1/stamp.jpg"", ""points"": []}",flagged/Detected Instances/fe42b78529616b432d55/image.webp,45,,,2024-06-25 12:43:45.012330 +flagged/Input Image/37cf51da176e45f176c8/lego.jpg,yellow,"{""image"": ""flagged/CLICK DRAG to specify object to count with visual exemplars here/beeb08664504d784a62e/lego.jpg"", ""points"": []}",flagged/Detected Instances/c420149b5762709cd085/image.webp,23,,,2024-06-25 12:44:27.177000 +flagged/Input Image/96219bcbf20113a0aab8/lego.jpg,studs,"{""image"": ""flagged/CLICK DRAG to specify object to count with visual exemplars here/699acba2f203aa14fc16/lego.jpg"", ""points"": []}",flagged/Detected Instances/18d8ce74eab9759ffa86/image.webp,23,,,2024-06-25 12:44:37.787345 +flagged/Input Image/3fa513923f4a4cbc1650/strawberry.jpg,strawberry,"{""image"": ""flagged/CLICK DRAG to specify object to count with visual exemplars here/b26302259971622261f7/strawberry.jpg"", ""points"": []}",flagged/Detected Instances/8d551fe44abbcf336b71/image.webp,16,,,2024-06-25 12:45:02.003915 +flagged/Input Image/d9e2d9688dc9a82fa656/strawberry.jpg,elderberry,"{""image"": ""flagged/CLICK DRAG to specify object to count with visual exemplars here/f86eed288abddd19b0d2/strawberry.jpg"", ""points"": []}",flagged/Detected Instances/cc78b0d8c3d24a3b12e7/image.webp,0,,,2024-06-25 12:45:16.103419 +flagged/Input Image/d74aa8ff2ec0dbba88b7/strawberry.jpg,blueberry,"{""image"": ""flagged/CLICK DRAG to specify object to count with visual exemplars here/89b0adc1664a53a48fea/strawberry.jpg"", ""points"": []}",flagged/Detected Instances/b97b8f4849493ce67de8/image.webp,137,,,2024-06-25 12:45:29.869576 +flagged/Input Image/84eb8736bc8eb48f507f/strawberry.jpg,blueberry,"{""image"": ""flagged/CLICK DRAG to specify object to count with visual exemplars here/0978ec84da8050ca0792/strawberry.jpg"", ""points"": [[330.0, 158.0, 2.0, 354.0, 195.0, 3.0], [289.0, 248.0, 2.0, 307.0, 284.0, 3.0]]}",flagged/Detected Instances/923571e5c8db86d8a8ed/image.webp,132,,,2024-06-25 12:46:16.470770 +flagged/Input Image/104d1b72504a1965f84d/fish.jpg,fish,"{""image"": ""flagged/CLICK DRAG to specify object to count with visual exemplars here/1e96cf74eb0dea109d9f/fish.jpg"", ""points"": []}",flagged/Detected Instances/3df418860ba1a58017cc/image.webp,248,,,2024-06-25 12:46:35.001964 +flagged/Input Image/6ad4c8683b86783f3ec2/stamp.jpg,square,"{""image"": ""flagged/CLICK DRAG to specify object to count with visual exemplars here/1eefba3eed3b8e2a271b/stamp.jpg"", ""points"": [[300.0, 285.0, 2.0, 351.0, 327.0, 3.0], [240.0, 337.0, 2.0, 290.0, 379.0, 3.0]]}",flagged/Detected Instances/eaab5820db88aac49462/image.webp,45,,,2024-06-25 12:47:11.901813 +flagged/Input Image/9eb585667d874ba79393/stamp.jpg,red rectangular stamp,"{""image"": ""flagged/CLICK DRAG to specify object to count with visual exemplars here/88c962fe4751ec3839cd/stamp.jpg"", ""points"": [[300.0, 285.0, 2.0, 351.0, 327.0, 3.0], [240.0, 337.0, 2.0, 290.0, 379.0, 3.0]]}",flagged/Detected Instances/cd25d7af77d301a1d5a4/image.webp,45,,,2024-06-25 12:49:38.403498 +flagged/Input Image/9724ddc2f18fd68bdee9/stamp.jpg,red ,"{""image"": ""flagged/CLICK DRAG to specify object to count with visual exemplars here/13e9c5b34604bb92d7fc/stamp.jpg"", ""points"": [[300.0, 285.0, 2.0, 351.0, 327.0, 3.0], [240.0, 337.0, 2.0, 290.0, 379.0, 3.0]]}",flagged/Detected Instances/57bf0944ab05bd0099ea/image.webp,45,,,2024-06-25 12:49:51.522005 +flagged/Input Image/1238680c118136bb0496/lego.jpg,red lego,"{""image"": ""flagged/CLICK DRAG to specify object to count with visual exemplars here/05be72fc352ad2ba08ba/lego.jpg"", ""points"": []}",flagged/Detected Instances/11c893cbcf45bcf1ab51/image.webp,23,,,2024-06-25 12:50:27.240856 +flagged/Input Image/a305b9fbd30361d8d3bf/lego.jpg,red ,"{""image"": ""flagged/CLICK DRAG to specify object to count with visual exemplars here/a7ebb8ee6c2ad9540939/lego.jpg"", ""points"": []}",flagged/Detected Instances/e5780fbd02076a52113b/image.webp,23,,,2024-06-25 12:50:37.774851 +flagged/Input Image/34d0773dc8f14c58f797/lego.jpg,white,"{""image"": ""flagged/CLICK DRAG to specify object to count with visual exemplars here/23ead2166dba8533b130/lego.jpg"", ""points"": []}",flagged/Detected Instances/91146d3e4f8f27b15856/image.webp,23,,,2024-06-25 12:50:50.401975 +flagged/Input Image/c76b4273734935f64ecf/strawberry.jpg,strawberry,"{""image"": ""flagged/CLICK DRAG to specify object to count with visual exemplars here/ee8ba6f40aa26158d870/strawberry.jpg"", ""points"": []}",flagged/Detected Instances/d1e59cae00d004db8b42/image.webp,16,,,2024-06-25 12:51:05.753104 +flagged/Input Image/a6c4912aa51cc54ed3d7/strawberry.jpg,blueberry,"{""image"": ""flagged/CLICK DRAG to specify object to count with visual exemplars here/566190bf535ad3281f6c/strawberry.jpg"", ""points"": []}",flagged/Detected Instances/0a2ffbe024c77e7da50d/image.webp,137,,,2024-06-25 12:51:31.117635 +flagged/Input Image/ea56a8e9ad4e135ff663/strawberry.jpg,blueberry,"{""image"": ""flagged/CLICK DRAG to specify object to count with visual exemplars here/c92e960c7b3df79ff7b5/strawberry.jpg"", ""points"": [[341.0, 123.0, 2.0, 361.0, 149.0, 3.0]]}",flagged/Detected Instances/428433149285aee052cf/image.webp,134,,,2024-06-25 12:51:42.720447 +flagged/Input Image/6fc4ed98f8495057126d/women.jpg,girl,"{""image"": ""flagged/CLICK DRAG to specify object to count with visual exemplars here/ec38f075c8bc9249bfb6/women.jpg"", ""points"": []}",flagged/Detected Instances/bc7f57f26e088938c86a/image.webp,5,,,2024-06-25 12:52:16.523201 +flagged/Input Image/8d06bc9272737637658c/women.jpg,boy,"{""image"": ""flagged/CLICK DRAG to specify object to count with visual exemplars here/94a9eef129e73dec7ed8/women.jpg"", ""points"": []}",flagged/Detected Instances/c419565161b7941b9368/image.webp,1,,,2024-06-25 12:52:34.380793 +flagged/Input Image/18639d94ba6ee7673454/fish.jpg,fish,"{""image"": ""flagged/CLICK DRAG to specify object to count with visual exemplars here/4d917d8cff994b9757bd/fish.jpg"", ""points"": []}",flagged/Detected Instances/43bbba43eb4a991b6256/image.webp,248,,,2024-06-25 12:53:25.668942 +flagged/Input Image/acbbf9f607925c34aec9/strawberry.jpg,strawberry,"{""image"": ""flagged/Specify object to count with visual exemplars here/a66486911b04c6d3dd5f/strawberry.jpg"", ""points"": []}",flagged/Detected Instances/71aa71914a32c98dbf5e/image.webp,16,,,2024-06-25 13:21:35.534920 +flagged/Input Image/5a125c995d0990aaf69b/bird-1.JPG,bird,"{""image"": ""flagged/Specify object to count with visual exemplars here/c8b24b1c09de0bbcfa35/bird-2.JPG"", ""points"": []}",flagged/Detected Instances/a24ce3527db178c02415/image.webp,30,,,2024-06-25 13:24:10.446815 +flagged/Input Image/be297ebfd1bb43cd1438/strawberry.jpg,strawberry,"{""image"": ""flagged/Specify object to count with visual exemplars here/d6e9603b839cb704d1e1/strawberry.jpg"", ""points"": []}",flagged/Detected Instances/ab49e40bc9aefd394529/image.webp,16,,,2024-06-25 13:27:08.069932 +flagged/Input Image/667eae2d1cf1f7cb10bf/apple.jpg,apple,"{""image"": ""flagged/Specify object to count with visual exemplars here/d4e8125f9602a8823edd/apple.jpg"", ""points"": []}",flagged/Detected Instances/9568219f942093ef51cf/image.webp,568,,,2024-06-25 13:47:38.683788 +flagged/Input Image/ff98881ffa5a95d08f37/women.jpg,girl,"{""image"": ""flagged/Specify object to count with visual exemplars here/8afee605e88ec6ddaa37/women.jpg"", ""points"": []}",flagged/Detected Instances/9c09433356e51d42d738/image.webp,5,,,2024-06-25 13:47:53.014232 +flagged/Input Image/0c059862756e0017eb5b/deer.jpg,deer,"{""image"": ""flagged/Specify object to count with visual exemplars here/454665f365b4cd0826e9/deer.jpg"", ""points"": []}",flagged/Detected Instances/95bf56b70e2f2e6f3fe6/image.webp,21,,,2024-06-25 13:48:07.565103 +flagged/Input Image/59a8de455bfa9769d8f0/egg.jpg,egg,"{""image"": ""flagged/Specify object to count with visual exemplars here/6626b7927b70e282f908/egg.jpg"", ""points"": []}",flagged/Detected Instances/043ac97ae83b87ef6d7d/image.webp,18,,,2024-06-25 13:48:24.214285 +flagged/Input Image/f916803fd0379cb3b3b6/women.jpg,foot,"{""image"": ""flagged/Specify object to count with visual exemplars here/e743db45e1b6c6dbce71/women.jpg"", ""points"": []}",flagged/Detected Instances/9906067993749438f371/image.webp,0,,,2024-06-25 13:52:46.735106 +flagged/Input Image/b014d0dd065e2cc85351/women.jpg,shoe,"{""image"": ""flagged/Specify object to count with visual exemplars here/15cc3b4b2ee3fa63144b/women.jpg"", ""points"": []}",flagged/Detected Instances/2a54e56bffa5f3c38df3/image.webp,13,,,2024-06-25 13:52:55.287490 +flagged/Input Image/e58990d9da6323e900ec/strawberry.jpg,blueberry,"{""image"": ""flagged/Specify object to count with visual exemplars here/f9d6362f7f07cc8455ba/strawberry.jpg"", ""points"": []}",flagged/Detected Instances/962b404f01817d4b4240/image.webp,137,,,2024-06-25 13:53:24.831821 +flagged/Input Image/36b16cc3616ed8f56ff1/strawberry.jpg,blueberry,"{""image"": ""flagged/Specify object to count with visual exemplars here/c70c72e5fad645bc14aa/strawberry.jpg"", ""points"": [[300.0, 97.0, 2.0, 321.0, 122.0, 3.0]]}",flagged/Detected Instances/c8d02635413c34abe79e/image.webp,134,,,2024-06-25 13:53:33.382644 +flagged/Input Image/abf2935032503027b2a6/strawberry.jpg,,"{""image"": ""flagged/Specify object to count with visual exemplars here/9d3f4b6f03f0dca04bc9/strawberry.jpg"", ""points"": [[300.0, 97.0, 2.0, 321.0, 122.0, 3.0], [165.0, 74.0, 2.0, 204.0, 133.0, 3.0], [104.0, 182.0, 2.0, 151.0, 230.0, 3.0]]}",flagged/Detected Instances/0e52e648edb9fd8db2d5/image.webp,143,,,2024-06-25 13:54:00.465000 +flagged/Input Image/af6c4808a13098046947/strawberry.jpg,,"{""image"": ""flagged/Specify object to count with visual exemplars here/75ed6a32a8a1e6340110/strawberry.jpg"", ""points"": [[99.0, 139.0, 2.0, 146.0, 185.0, 3.0], [161.0, 71.0, 2.0, 202.0, 132.0, 3.0]]}",flagged/Detected Instances/db51c4253db3807889f0/image.webp,16,,,2024-06-25 13:54:22.839814 +flagged/Input Image/c716b9c2502379b4f53d/strawberry.jpg,strawberry,"{""image"": ""flagged/Specify object to count with visual exemplars here/1ab41b769030db4ba746/strawberry.jpg"", ""points"": []}",flagged/Detected Instances/1d060462444efcff7c1b/image.webp,16,,,2024-06-25 14:24:43.543530 +flagged/Input Image/aca09cc83af42dc6a5ed/women.jpg,girl,"{""image"": ""flagged/Specify object to count with visual exemplars here/3399db8186d98f479b4f/women.jpg"", ""points"": []}",flagged/Detected Instances/a878c25daaecd95cd2e3/image.webp,5,,,2024-06-25 14:26:04.205110 +flagged/Input Image/f0e3038613138f5653fe/fish.jpg,fish,"{""image"": ""flagged/Specify object to count with visual exemplars here/5a2fbfebe22860e70887/fish.jpg"", ""points"": []}",flagged/Detected Instances/b00af2ba550cbce038ed/image.webp,248,,,2024-06-25 14:41:36.984087 +flagged/Input Image/f22235f8e59c137765af/deer.jpg,deer,"{""image"": ""flagged/Specify object to count with visual exemplars here/79acfe92d19d88df7924/deer.jpg"", ""points"": []}",flagged/Detected Instances/43e430b66d190a4be7b6/image.webp,21,,,2024-06-25 14:42:57.479610 +flagged/Input Image/43828e990ec1f6c5dc14/fish.jpg,fish,"{""image"": ""flagged/Specify object to count with visual exemplars here/ee76194fb63611ab5b90/fish.jpg"", ""points"": []}",flagged/Detected Instances/04969bb2604f9248742b/image.webp,248,,,2024-06-25 14:45:20.424378 +flagged/Input Image/1e2bb41e969d56102dc2/bird-1.JPG,bird,"{""image"": ""flagged/Specify object to count with visual exemplars here/01989c9431d1e6c77ad1/bird-2.JPG"", ""points"": []}",flagged/Detected Instances/4957f1a733161da1ddcd/image.webp,30,,,2024-06-25 14:48:39.348196 +flagged/Input Image/e1f55639b7c03f2c3bd1/deer.jpg,deer,"{""image"": ""flagged/Specify object to count with visual exemplars here/7207f6ffa7c700e4339b/deer.jpg"", ""points"": []}",flagged/Detected Instances/c64f5cf630c80bfc90b5/image.webp,21,,,2024-06-25 14:48:56.962720 +flagged/Input Image/adef57f83578033397ef/strawberry.jpg,strawberry,"{""image"": ""flagged/Specify object to count with visual exemplars here/a77ac336261b1cb69b31/strawberry.jpg"", ""points"": []}",flagged/Detected Instances/99b7039bc665946889cf/image.webp,16,,,2024-06-25 14:49:23.944300 +flagged/Input Image/fad7be92b6af8fc44818/strawberry.jpg,strawberry,"{""image"": ""flagged/Specify object to count with visual exemplars here/fbd49488b1c780e4d5d1/strawberry.jpg"", ""points"": []}",flagged/Detected Instances/26fa7fa2475e60ea3221/image.webp,16,,,2024-06-25 14:51:48.832356 +flagged/Input Image/ab0e307953f58465f254/strawberry.jpg,strawberry,"{""image"": ""flagged/Specify object to count with visual exemplars here/162fa100594505627551/strawberry.jpg"", ""points"": []}",flagged/Detected Instances/4351f6b9dc586a7a192f/image.webp,16,,,2024-06-25 14:53:19.317939 +flagged/Input Image/64cf922799f8f994f127/strawberry.jpg,strawberry,"{""image"": ""flagged/Specify object to count with visual exemplars here/be38b70786240819da5e/strawberry.jpg"", ""points"": [[183.0, 131.0, 1.0, 0.0, 0.0, 4.0], [272.0, 124.0, 1.0, 0.0, 0.0, 4.0], [272.0, 124.0, 2.0, 325.0, 186.0, 3.0]]}",flagged/Detected Instances/7e5c71b26561217570d0/image.webp,16,,,2024-06-25 14:53:37.751975 +flagged/Input Image/7a7dacbf8c123986c050/strawberry.jpg,strawberry,"{""image"": ""flagged/Specify object to count with visual exemplars here/50f292a91824f56a78ec/strawberry.jpg"", ""points"": [[183.0, 131.0, 1.0, 0.0, 0.0, 4.0], [272.0, 124.0, 1.0, 0.0, 0.0, 4.0], [272.0, 124.0, 2.0, 325.0, 186.0, 3.0], [335.0, 170.0, 1.0, 0.0, 0.0, 4.0], [286.0, 172.0, 1.0, 0.0, 0.0, 4.0], [275.0, 193.0, 1.0, 0.0, 0.0, 4.0], [267.0, 207.0, 2.0, 285.0, 254.0, 3.0], [317.0, 223.0, 2.0, 345.0, 277.0, 3.0]]}",flagged/Detected Instances/cdeab185b16c9564290b/image.webp,150,,,2024-06-25 14:53:51.580488 +flagged/Input Image/4a1ed2bed23e714c2e98/fish.jpg,fish,"{""image"": ""flagged/Specify object to count with visual exemplars here/7b43211f1ed9441e8e60/fish.jpg"", ""points"": [[526.0, 469.0, 2.0, 577.0, 520.0, 3.0]]}",flagged/Detected Instances/b6581780403fac1fe05a/image.webp,247,,,2024-06-25 14:54:23.273247 +flagged/Input Image/c82e8b7773224a87ab1a/strawberry.jpg,blueberry,"{""image"": ""flagged/Specify object to count with visual exemplars here/3f1c6c4b8fa8b0faec98/strawberry.jpg"", ""points"": []}",flagged/Detected Instances/f9ffff38f04c59e61004/image.webp,137,,,2024-06-25 14:56:52.255423 +flagged/Input Image/5e7b302c38bef4e986e9/strawberry.jpg,blueberry,"{""image"": ""flagged/Specify object to count with visual exemplars here/296b7b929afa36c48677/strawberry.jpg"", ""points"": [[329.0, 168.0, 2.0, 353.0, 190.0, 3.0]]}",flagged/Detected Instances/1a9940b04abede2a55a9/image.webp,133,,,2024-06-25 14:57:03.171030 +flagged/Input Image/288e68a5d5996e695512/strawberry.jpg,strawberry,"{""image"": ""flagged/Specify object to count with visual exemplars here/e7aa3e19a9c873c247b5/strawberry.jpg"", ""points"": [[208.0, 92.0, 2.0, 236.0, 139.0, 3.0]]}",flagged/Detected Instances/062ea676672533bd63af/image.webp,16,,,2024-06-25 15:01:55.834836 +flagged/Input Image/65771625dd3209ab4b5e/women.jpg,girl,"{""image"": ""flagged/Specify object to count with visual exemplars here/9cc7376257b49f9a02ef/women.jpg"", ""points"": [[267.0, 320.0, 2.0, 300.0, 362.0, 3.0]]}",flagged/Detected Instances/3e86fdbcad5cb3dbab9a/image.webp,6,,,2024-06-25 15:02:27.623802 +flagged/Input Image/6d2d8805808abba5c681/strawberry.jpg,strawberry,"{""image"": ""flagged/Specify object to count with visual exemplars here/9f49a41bd458e9c37e95/strawberry.jpg"", ""points"": [[305.0, 187.0, 1.0, 0.0, 0.0, 4.0], [166.0, 56.0, 2.0, 203.0, 136.0, 3.0]]}",flagged/Detected Instances/9618d50e52b98e8ecfd1/image.webp,16,,,2024-06-25 15:08:35.121868 +flagged/Input Image/12e08ca8d302970a2008/strawberry.jpg,strawberry,"{""image"": ""flagged/Specify object to count with visual exemplars here/ee9d14118e586de08ead/strawberry.jpg"", ""points"": [[305.0, 187.0, 1.0, 0.0, 0.0, 4.0], [166.0, 56.0, 2.0, 203.0, 136.0, 3.0], [301.0, 96.0, 2.0, 319.0, 118.0, 3.0], [313.0, 165.0, 2.0, 333.0, 184.0, 3.0], [271.0, 282.0, 2.0, 285.0, 309.0, 3.0], [301.0, 242.0, 2.0, 320.0, 262.0, 3.0], [329.0, 162.0, 2.0, 358.0, 185.0, 3.0]]}",flagged/Detected Instances/05fd3125ccda9546e2b6/image.webp,146,,,2024-06-25 15:08:50.604743 +flagged/Input Image/4e1dd60a00267ba7ac80/women.jpg,boy,"{""image"": ""flagged/Specify object to count with visual exemplars here/4a178f28ec8e9901388e/women.jpg"", ""points"": []}",flagged/Detected Instances/b7d4a5530dfdf01f426a/image.webp,1,,,2024-06-25 15:09:42.749784 +flagged/Input Image/813b6b4e4a70fea8e13c/egg.jpg,egg,"{""image"": ""flagged/Specify object to count with visual exemplars here/9fc0bb0e2e5ed2a7c023/egg.jpg"", ""points"": []}",flagged/Detected Instances/6a7dbfbbbf65a567057e/image.webp,18,,,2024-06-25 15:09:55.901192 +flagged/Input Image/c09c36810b55389abddc/lego.jpg,yellow lego stud,"{""image"": ""flagged/Specify object to count with visual exemplars here/696b3fd37ad6c635baba/lego.jpg"", ""points"": []}",flagged/Detected Instances/60c9fcde0bb55251c8f1/image.webp,23,,,2024-06-25 15:10:20.114086 +flagged/Input Image/bd286866d8e9a3e48bad/lego.jpg,red,"{""image"": ""flagged/Specify object to count with visual exemplars here/719beaa78bcd4eb92c8b/lego.jpg"", ""points"": []}",flagged/Detected Instances/87cfe221a685444fa193/image.webp,23,,,2024-06-25 15:10:33.233397 +flagged/Input Image/89c65135bd1fcf0062af/green-pea.jpg,green pea,"{""image"": ""flagged/Specify object to count with visual exemplars here/b2de1a09bc58b6b4d3ae/green-pea.jpg"", ""points"": []}",flagged/Detected Instances/7272d7039ec368ccb1ac/image.webp,213,,,2024-06-25 15:10:42.163670 +flagged/Input Image/a6c33d85c85f75ef8866/stamp.jpg,stamp,"{""image"": ""flagged/Specify object to count with visual exemplars here/679009966f23256eed77/stamp.jpg"", ""points"": []}",flagged/Detected Instances/946f4a626a568a22de21/image.webp,45,,,2024-06-25 15:10:57.228588 +flagged/Input Image/5954d7cfd21da1b1d083/webcam.png,girl,"{""image"": ""flagged/Specify object to count with visual exemplars here/406bdbe8b94e97f309e1/stamp.jpg"", ""points"": []}",flagged/Detected Instances/671208d6944225443700/image.webp,1,,,2024-06-25 15:13:06.226068 +flagged/Input Image/0d653baaf7b60d4fa9a9/webcam.png,,"{""image"": ""flagged/Specify object to count with visual exemplars here/4961820b4b19fa1543ab/webcam.png"", ""points"": [[348.0, 198.0, 2.0, 825.0, 594.0, 3.0]]}",flagged/Detected Instances/11d3ce706b7c55e4b29d/image.webp,6,,,2024-06-25 15:13:46.575188 +flagged/Input Image/2fc47796e17477b72a7b/webcam.png,niki,"{""image"": ""flagged/Specify object to count with visual exemplars here/ed991be65bcc28198008/webcam.png"", ""points"": [[348.0, 198.0, 2.0, 825.0, 594.0, 3.0]]}",flagged/Detected Instances/4b4c120051c76b356ad9/image.webp,11,,,2024-06-25 15:14:11.677104 +flagged/Input Image/c97af0e3fe6b528a4787/webcam.png,girl,"{""image"": ""flagged/Specify object to count with visual exemplars here/b2e5b5ad196b52f66b65/webcam.png"", ""points"": [[348.0, 198.0, 2.0, 825.0, 594.0, 3.0]]}",flagged/Detected Instances/99c1019f28a1ea86e661/image.webp,5,,,2024-06-25 15:14:22.291803 +flagged/Input Image/6d42f7157ab42c91e2d7/webcam.png,girl,"{""image"": ""flagged/Specify object to count with visual exemplars here/7fb61e9fc487a3698878/webcam.png"", ""points"": []}",flagged/Detected Instances/bbe44c1a92d592ae7d96/image.webp,1,,,2024-06-25 15:14:34.641650 +flagged/Input Image/cd2fd26787e15861e3d8/apple.jpg,apple,"{""image"": ""flagged/Specify object to count with visual exemplars here/67cf19d7f6ecdd3495a0/apple.jpg"", ""points"": []}",flagged/Detected Instances/c828265963027a846279/image.webp,568,,,2024-06-25 15:39:38.657610 +flagged/Input Image/676bdd4d886064a651ff/apple.jpg,apple,"{""image"": ""flagged/Specify object to count with visual exemplars here/26e0d8f5fa299f54b2dd/apple.jpg"", ""points"": [[316.0, 184.0, 2.0, 371.0, 220.0, 3.0]]}",flagged/Detected Instances/f281b9015813f6849ef0/image.webp,541,,,2024-06-25 15:40:22.068821 +flagged/Input Image/157f9999f8285255f079/apple.jpg,apple,"{""image"": ""flagged/Specify object to count with visual exemplars here/9f08512a7bc78db40a16/apple.jpg"", ""points"": [[316.0, 184.0, 2.0, 371.0, 220.0, 3.0], [118.0, 245.0, 2.0, 172.0, 276.0, 3.0]]}",flagged/Detected Instances/20546e887359f9ab6311/image.webp,552,,,2024-06-25 15:40:31.337257 +flagged/Input Image/dfefe4e8ad7004a21057/apple.jpg,apple,"{""image"": ""flagged/Specify object to count with visual exemplars here/0e01c3339d835fc181f9/apple.jpg"", ""points"": [[316.0, 184.0, 2.0, 371.0, 220.0, 3.0], [118.0, 245.0, 2.0, 172.0, 276.0, 3.0], [314.0, 230.0, 2.0, 357.0, 265.0, 3.0]]}",flagged/Detected Instances/00f422daee514dc14f95/image.webp,540,,,2024-06-25 15:40:39.970940 +flagged/Input Image/740c54384733c6909520/deer.jpg,deer,"{""image"": ""flagged/Specify object to count with visual exemplars here/20146d57e02b87830aeb/deer.jpg"", ""points"": []}",flagged/Detected Instances/5f4bb2b3b91635f81795/image.webp,21,,,2024-06-25 15:40:53.851874 +flagged/Input Image/5a9d46d5f1fb47ee307b/orchard.jpg,apple,"{""image"": ""flagged/Specify object to count with visual exemplars here/6d352f6e7b5a6c866fdb/deer.jpg"", ""points"": []}",flagged/Detected Instances/6c2b985de60b38896d84/image.webp,543,,,2024-06-25 15:42:01.647977 +flagged/Input Image/de66ddbd6870a9789205/webcam.png,bunny,"{""image"": ""flagged/Specify object to count with visual exemplars here/cd0638cb70d92342ecda/deer.jpg"", ""points"": []}",flagged/Detected Instances/dd795b53a5c38aa3207c/image.webp,0,,,2024-06-25 15:43:54.093401 +flagged/Input Image/4193d822e105ac4695ff/webcam.png,man,"{""image"": ""flagged/Specify object to count with visual exemplars here/13affadc040616f325fb/deer.jpg"", ""points"": []}",flagged/Detected Instances/deb096b5bb87593f8894/image.webp,1,,,2024-06-25 15:44:03.789939 +flagged/Input Image/5dca2e19924d68ff12c5/webcam.png,yoga mat,"{""image"": ""flagged/Specify object to count with visual exemplars here/3c08834b2231ee8173b6/deer.jpg"", ""points"": []}",flagged/Detected Instances/4ccd7a4d047f6cc4f438/image.webp,0,,,2024-06-25 15:44:18.374825 +flagged/Input Image/7dc8361a1e198ce87161/webcam.png,microwave,"{""image"": ""flagged/Specify object to count with visual exemplars here/2ab8fa7854e2e8f488b7/deer.jpg"", ""points"": []}",flagged/Detected Instances/5a1c457670ba55bd7f22/image.webp,1,,,2024-06-25 15:44:29.556338 +flagged/Input Image/e110ca4a4a40edaa6c2f/webcam.png,eyebrow,"{""image"": ""flagged/Specify object to count with visual exemplars here/2b5d4f4150c5af42be5e/deer.jpg"", ""points"": []}",flagged/Detected Instances/e6e0f246c363ee76f378/image.webp,0,,,2024-06-25 15:44:54.438375 +flagged/Input Image/b9323d538671be8fb66c/webcam.png,ear,"{""image"": ""flagged/Specify object to count with visual exemplars here/d218867fab61d8837d1b/deer.jpg"", ""points"": []}",flagged/Detected Instances/8d6a4f375884a2862603/image.webp,0,,,2024-06-25 15:45:04.003679 +Submit,Clear,flagged/Input Image/207cd608b7c1a6c526bd/strawberry.jpg,strawberry,"{""image"": ""flagged/Specify object to count with visual exemplars here/4dfe49b3bb9109a5112c/strawberry.jpg"", ""points"": []}",flagged/Detected Instances/69dbd8ce1b54204aa311/image.webp,16,,,2024-06-25 17:17:28.470393 +Submit,Clear,flagged/Input Image/ccb9fedaf141f3cb6f69/strawberry.jpg,strawberry,"{""image"": ""flagged/Specify object to count with visual exemplars here/9f5e5147ae36d6c07538/strawberry.jpg"", ""points"": []}",flagged/Detected Instances/cd6f4c50bdf9f10604db/image.webp,16,,,2024-06-25 17:48:13.969532 diff --git a/flagged/output/a686c9813d1a6c436907/image.webp b/flagged/output/a686c9813d1a6c436907/image.webp new file mode 100644 index 0000000000000000000000000000000000000000..2900972d92568400f21ef4a2e73770be0a31f39a Binary files /dev/null and b/flagged/output/a686c9813d1a6c436907/image.webp differ diff --git a/gradio-demo-img.png b/gradio-demo-img.png new file mode 100644 index 0000000000000000000000000000000000000000..af441780c72c66afd21530f8ecd802ef8e8e78ba --- /dev/null +++ b/gradio-demo-img.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:09bd972ca4a129ed5cf011251e8d63ec4c9e97a2b9641d4fa871b550b41aece6 +size 1539686 diff --git a/green-pea.jpg b/green-pea.jpg new file mode 100644 index 0000000000000000000000000000000000000000..fa36364f74b98e909a3d634d2d39e4c98885f206 Binary files /dev/null and b/green-pea.jpg differ diff --git a/groundingdino/util/__init__.py b/groundingdino/util/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..168f9979a4623806934b0ff1102ac166704e7dec --- /dev/null +++ b/groundingdino/util/__init__.py @@ -0,0 +1 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved diff --git a/groundingdino/util/__pycache__/__init__.cpython-38.pyc b/groundingdino/util/__pycache__/__init__.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f26bbb267a9dedf364cd3a8b833f3aa32c0d9599 Binary files /dev/null and b/groundingdino/util/__pycache__/__init__.cpython-38.pyc differ diff --git a/groundingdino/util/__pycache__/__init__.cpython-39.pyc b/groundingdino/util/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f049f4fb58e1b97ff2daecb791afd163dd8eec07 Binary files /dev/null and b/groundingdino/util/__pycache__/__init__.cpython-39.pyc differ diff --git a/groundingdino/util/__pycache__/box_ops.cpython-38.pyc b/groundingdino/util/__pycache__/box_ops.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2f1838d21cc372095140a6891f874be8e9718978 Binary files /dev/null and b/groundingdino/util/__pycache__/box_ops.cpython-38.pyc differ diff --git a/groundingdino/util/__pycache__/box_ops.cpython-39.pyc b/groundingdino/util/__pycache__/box_ops.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ac82f180fda8dd659f76d31e1e4ffa64d11c67cf Binary files /dev/null and b/groundingdino/util/__pycache__/box_ops.cpython-39.pyc differ diff --git a/groundingdino/util/__pycache__/get_tokenlizer.cpython-38.pyc b/groundingdino/util/__pycache__/get_tokenlizer.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6cd105f5013beb031e3f166bfdc193c3e11c4e5e Binary files /dev/null and b/groundingdino/util/__pycache__/get_tokenlizer.cpython-38.pyc differ diff --git a/groundingdino/util/__pycache__/get_tokenlizer.cpython-39.pyc b/groundingdino/util/__pycache__/get_tokenlizer.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..03bf6342f705005b1edae2e4d6fed02019a211fd Binary files /dev/null and b/groundingdino/util/__pycache__/get_tokenlizer.cpython-39.pyc differ diff --git a/groundingdino/util/__pycache__/misc.cpython-38.pyc b/groundingdino/util/__pycache__/misc.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7c88693f813b87e94690dbf117547c483dd3a705 Binary files /dev/null and b/groundingdino/util/__pycache__/misc.cpython-38.pyc differ diff --git a/groundingdino/util/__pycache__/misc.cpython-39.pyc b/groundingdino/util/__pycache__/misc.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..cb07db9c3b32ab5e2d5cb7712aa76186977f82af Binary files /dev/null and b/groundingdino/util/__pycache__/misc.cpython-39.pyc differ diff --git a/groundingdino/util/__pycache__/slconfig.cpython-38.pyc b/groundingdino/util/__pycache__/slconfig.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e3520fc7264b58994828ba1d2bfc70d124154877 Binary files /dev/null and b/groundingdino/util/__pycache__/slconfig.cpython-38.pyc differ diff --git a/groundingdino/util/__pycache__/slconfig.cpython-39.pyc b/groundingdino/util/__pycache__/slconfig.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..546f2f0cf904e26ea97af3618e29da889cc50920 Binary files /dev/null and b/groundingdino/util/__pycache__/slconfig.cpython-39.pyc differ diff --git a/groundingdino/util/__pycache__/utils.cpython-38.pyc b/groundingdino/util/__pycache__/utils.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e3b5def89973b117f9f85c919facafc55b837958 Binary files /dev/null and b/groundingdino/util/__pycache__/utils.cpython-38.pyc differ diff --git a/groundingdino/util/__pycache__/utils.cpython-39.pyc b/groundingdino/util/__pycache__/utils.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..45ba3e6564acf7ceeaca3eeeb3e535abcb95dcb5 Binary files /dev/null and b/groundingdino/util/__pycache__/utils.cpython-39.pyc differ diff --git a/groundingdino/util/__pycache__/visualizer.cpython-38.pyc b/groundingdino/util/__pycache__/visualizer.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a60fccd2db6404400d508d6b2ad99dab7362970b Binary files /dev/null and b/groundingdino/util/__pycache__/visualizer.cpython-38.pyc differ diff --git a/groundingdino/util/__pycache__/visualizer.cpython-39.pyc b/groundingdino/util/__pycache__/visualizer.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b79551106767ca47af6a27b2dede53c3a0a7acda Binary files /dev/null and b/groundingdino/util/__pycache__/visualizer.cpython-39.pyc differ diff --git a/groundingdino/util/__pycache__/vl_utils.cpython-38.pyc b/groundingdino/util/__pycache__/vl_utils.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7ce5a3c87a94da3e0a37bfe3ba733627cf171dd8 Binary files /dev/null and b/groundingdino/util/__pycache__/vl_utils.cpython-38.pyc differ diff --git a/groundingdino/util/__pycache__/vl_utils.cpython-39.pyc b/groundingdino/util/__pycache__/vl_utils.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..252ee8ba3ce5af9c3e62f772c196b55bb0a7dd6c Binary files /dev/null and b/groundingdino/util/__pycache__/vl_utils.cpython-39.pyc differ diff --git a/groundingdino/util/box_ops.py b/groundingdino/util/box_ops.py new file mode 100644 index 0000000000000000000000000000000000000000..781068d294e576954edb4bd07b6e0f30e4e1bcd9 --- /dev/null +++ b/groundingdino/util/box_ops.py @@ -0,0 +1,140 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved +""" +Utilities for bounding box manipulation and GIoU. +""" +import torch +from torchvision.ops.boxes import box_area + + +def box_cxcywh_to_xyxy(x): + x_c, y_c, w, h = x.unbind(-1) + b = [(x_c - 0.5 * w), (y_c - 0.5 * h), (x_c + 0.5 * w), (y_c + 0.5 * h)] + return torch.stack(b, dim=-1) + + +def box_xyxy_to_cxcywh(x): + x0, y0, x1, y1 = x.unbind(-1) + b = [(x0 + x1) / 2, (y0 + y1) / 2, (x1 - x0), (y1 - y0)] + return torch.stack(b, dim=-1) + + +# modified from torchvision to also return the union +def box_iou(boxes1, boxes2): + area1 = box_area(boxes1) + area2 = box_area(boxes2) + + # import ipdb; ipdb.set_trace() + lt = torch.max(boxes1[:, None, :2], boxes2[:, :2]) # [N,M,2] + rb = torch.min(boxes1[:, None, 2:], boxes2[:, 2:]) # [N,M,2] + + wh = (rb - lt).clamp(min=0) # [N,M,2] + inter = wh[:, :, 0] * wh[:, :, 1] # [N,M] + + union = area1[:, None] + area2 - inter + + iou = inter / (union + 1e-6) + return iou, union + + +def generalized_box_iou(boxes1, boxes2): + """ + Generalized IoU from https://giou.stanford.edu/ + + The boxes should be in [x0, y0, x1, y1] format + + Returns a [N, M] pairwise matrix, where N = len(boxes1) + and M = len(boxes2) + """ + # degenerate boxes gives inf / nan results + # so do an early check + assert (boxes1[:, 2:] >= boxes1[:, :2]).all() + assert (boxes2[:, 2:] >= boxes2[:, :2]).all() + # except: + # import ipdb; ipdb.set_trace() + iou, union = box_iou(boxes1, boxes2) + + lt = torch.min(boxes1[:, None, :2], boxes2[:, :2]) + rb = torch.max(boxes1[:, None, 2:], boxes2[:, 2:]) + + wh = (rb - lt).clamp(min=0) # [N,M,2] + area = wh[:, :, 0] * wh[:, :, 1] + + return iou - (area - union) / (area + 1e-6) + + +# modified from torchvision to also return the union +def box_iou_pairwise(boxes1, boxes2): + area1 = box_area(boxes1) + area2 = box_area(boxes2) + + lt = torch.max(boxes1[:, :2], boxes2[:, :2]) # [N,2] + rb = torch.min(boxes1[:, 2:], boxes2[:, 2:]) # [N,2] + + wh = (rb - lt).clamp(min=0) # [N,2] + inter = wh[:, 0] * wh[:, 1] # [N] + + union = area1 + area2 - inter + + iou = inter / union + return iou, union + + +def generalized_box_iou_pairwise(boxes1, boxes2): + """ + Generalized IoU from https://giou.stanford.edu/ + + Input: + - boxes1, boxes2: N,4 + Output: + - giou: N, 4 + """ + # degenerate boxes gives inf / nan results + # so do an early check + assert (boxes1[:, 2:] >= boxes1[:, :2]).all() + assert (boxes2[:, 2:] >= boxes2[:, :2]).all() + assert boxes1.shape == boxes2.shape + iou, union = box_iou_pairwise(boxes1, boxes2) # N, 4 + + lt = torch.min(boxes1[:, :2], boxes2[:, :2]) + rb = torch.max(boxes1[:, 2:], boxes2[:, 2:]) + + wh = (rb - lt).clamp(min=0) # [N,2] + area = wh[:, 0] * wh[:, 1] + + return iou - (area - union) / area + + +def masks_to_boxes(masks): + """Compute the bounding boxes around the provided masks + + The masks should be in format [N, H, W] where N is the number of masks, (H, W) are the spatial dimensions. + + Returns a [N, 4] tensors, with the boxes in xyxy format + """ + if masks.numel() == 0: + return torch.zeros((0, 4), device=masks.device) + + h, w = masks.shape[-2:] + + y = torch.arange(0, h, dtype=torch.float) + x = torch.arange(0, w, dtype=torch.float) + y, x = torch.meshgrid(y, x) + + x_mask = masks * x.unsqueeze(0) + x_max = x_mask.flatten(1).max(-1)[0] + x_min = x_mask.masked_fill(~(masks.bool()), 1e8).flatten(1).min(-1)[0] + + y_mask = masks * y.unsqueeze(0) + y_max = y_mask.flatten(1).max(-1)[0] + y_min = y_mask.masked_fill(~(masks.bool()), 1e8).flatten(1).min(-1)[0] + + return torch.stack([x_min, y_min, x_max, y_max], 1) + + +if __name__ == "__main__": + x = torch.rand(5, 4) + y = torch.rand(3, 4) + iou, union = box_iou(x, y) + import ipdb + + ipdb.set_trace() diff --git a/groundingdino/util/get_tokenlizer.py b/groundingdino/util/get_tokenlizer.py new file mode 100644 index 0000000000000000000000000000000000000000..77873bcf4e098c194ef6182b8cbfdc2aa4e3d0ed --- /dev/null +++ b/groundingdino/util/get_tokenlizer.py @@ -0,0 +1,29 @@ +from transformers import AutoTokenizer, BertModel, BertTokenizer, RobertaModel, RobertaTokenizerFast +import os + +def get_tokenlizer(text_encoder_type): + if not isinstance(text_encoder_type, str): + # print("text_encoder_type is not a str") + if hasattr(text_encoder_type, "text_encoder_type"): + text_encoder_type = text_encoder_type.text_encoder_type + elif text_encoder_type.get("text_encoder_type", False): + text_encoder_type = text_encoder_type.get("text_encoder_type") + elif os.path.isdir(text_encoder_type) and os.path.exists(text_encoder_type): + pass + else: + raise ValueError( + "Unknown type of text_encoder_type: {}".format(type(text_encoder_type)) + ) + print("final text_encoder_type: {}".format(text_encoder_type)) + tokenizer = AutoTokenizer.from_pretrained(text_encoder_type) + print("load tokenizer done.") + return tokenizer + + +def get_pretrained_language_model(text_encoder_type): + if text_encoder_type == "bert-base-uncased" or (os.path.isdir(text_encoder_type) and os.path.exists(text_encoder_type)): + return BertModel.from_pretrained(text_encoder_type) + if text_encoder_type == "roberta-base": + return RobertaModel.from_pretrained(text_encoder_type) + + raise ValueError("Unknown text_encoder_type {}".format(text_encoder_type)) diff --git a/groundingdino/util/inference.py b/groundingdino/util/inference.py new file mode 100644 index 0000000000000000000000000000000000000000..d6e81d89db1c422bbf27e3c160ef0957dfa57223 --- /dev/null +++ b/groundingdino/util/inference.py @@ -0,0 +1,259 @@ +from typing import Tuple, List + +import cv2 +import numpy as np +import supervision as sv +import torch +from PIL import Image +from torchvision.ops import box_convert +import bisect + +import groundingdino.datasets.transforms as T +from groundingdino.models import build_model +from groundingdino.util.misc import clean_state_dict +from groundingdino.util.slconfig import SLConfig +from groundingdino.util.utils import get_phrases_from_posmap + +# ---------------------------------------------------------------------------------------------------------------------- +# OLD API +# ---------------------------------------------------------------------------------------------------------------------- + + +def preprocess_caption(caption: str) -> str: + result = caption.lower().strip() + if result.endswith("."): + return result + return result + "." + + +def load_model(model_config_path: str, model_checkpoint_path: str, device: str = "cuda"): + args = SLConfig.fromfile(model_config_path) + args.device = device + model = build_model(args) + checkpoint = torch.load(model_checkpoint_path, map_location="cpu") + model.load_state_dict(clean_state_dict(checkpoint["model"]), strict=False) + model.eval() + return model + + +def load_image(image_path: str) -> Tuple[np.array, torch.Tensor]: + transform = T.Compose( + [ + T.RandomResize([800], max_size=1333), + T.ToTensor(), + T.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]), + ] + ) + image_source = Image.open(image_path).convert("RGB") + image = np.asarray(image_source) + image_transformed, _ = transform(image_source, None) + return image, image_transformed + + +def predict( + model, + image: torch.Tensor, + caption: str, + box_threshold: float, + text_threshold: float, + device: str = "cuda", + remove_combined: bool = False +) -> Tuple[torch.Tensor, torch.Tensor, List[str]]: + caption = preprocess_caption(caption=caption) + + model = model.to(device) + image = image.to(device) + + with torch.no_grad(): + outputs = model(image[None], captions=[caption]) + + prediction_logits = outputs["pred_logits"].cpu().sigmoid()[0] # prediction_logits.shape = (nq, 256) + prediction_boxes = outputs["pred_boxes"].cpu()[0] # prediction_boxes.shape = (nq, 4) + + mask = prediction_logits.max(dim=1)[0] > box_threshold + logits = prediction_logits[mask] # logits.shape = (n, 256) + boxes = prediction_boxes[mask] # boxes.shape = (n, 4) + + tokenizer = model.tokenizer + tokenized = tokenizer(caption) + + if remove_combined: + sep_idx = [i for i in range(len(tokenized['input_ids'])) if tokenized['input_ids'][i] in [101, 102, 1012]] + + phrases = [] + for logit in logits: + max_idx = logit.argmax() + insert_idx = bisect.bisect_left(sep_idx, max_idx) + right_idx = sep_idx[insert_idx] + left_idx = sep_idx[insert_idx - 1] + phrases.append(get_phrases_from_posmap(logit > text_threshold, tokenized, tokenizer, left_idx, right_idx).replace('.', '')) + else: + phrases = [ + get_phrases_from_posmap(logit > text_threshold, tokenized, tokenizer).replace('.', '') + for logit + in logits + ] + + return boxes, logits.max(dim=1)[0], phrases + + +def annotate(image_source: np.ndarray, boxes: torch.Tensor, logits: torch.Tensor, phrases: List[str]) -> np.ndarray: + h, w, _ = image_source.shape + boxes = boxes * torch.Tensor([w, h, w, h]) + xyxy = box_convert(boxes=boxes, in_fmt="cxcywh", out_fmt="xyxy").numpy() + detections = sv.Detections(xyxy=xyxy) + + labels = [ + f"{phrase} {logit:.2f}" + for phrase, logit + in zip(phrases, logits) + ] + + box_annotator = sv.BoxAnnotator() + annotated_frame = cv2.cvtColor(image_source, cv2.COLOR_RGB2BGR) + annotated_frame = box_annotator.annotate(scene=annotated_frame, detections=detections, labels=labels) + return annotated_frame + + +# ---------------------------------------------------------------------------------------------------------------------- +# NEW API +# ---------------------------------------------------------------------------------------------------------------------- + + +class Model: + + def __init__( + self, + model_config_path: str, + model_checkpoint_path: str, + device: str = "cuda" + ): + self.model = load_model( + model_config_path=model_config_path, + model_checkpoint_path=model_checkpoint_path, + device=device + ).to(device) + self.device = device + + def predict_with_caption( + self, + image: np.ndarray, + caption: str, + box_threshold: float = 0.35, + text_threshold: float = 0.25 + ) -> Tuple[sv.Detections, List[str]]: + """ + import cv2 + + image = cv2.imread(IMAGE_PATH) + + model = Model(model_config_path=CONFIG_PATH, model_checkpoint_path=WEIGHTS_PATH) + detections, labels = model.predict_with_caption( + image=image, + caption=caption, + box_threshold=BOX_THRESHOLD, + text_threshold=TEXT_THRESHOLD + ) + + import supervision as sv + + box_annotator = sv.BoxAnnotator() + annotated_image = box_annotator.annotate(scene=image, detections=detections, labels=labels) + """ + processed_image = Model.preprocess_image(image_bgr=image).to(self.device) + boxes, logits, phrases = predict( + model=self.model, + image=processed_image, + caption=caption, + box_threshold=box_threshold, + text_threshold=text_threshold, + device=self.device) + source_h, source_w, _ = image.shape + detections = Model.post_process_result( + source_h=source_h, + source_w=source_w, + boxes=boxes, + logits=logits) + return detections, phrases + + def predict_with_classes( + self, + image: np.ndarray, + classes: List[str], + box_threshold: float, + text_threshold: float + ) -> sv.Detections: + """ + import cv2 + + image = cv2.imread(IMAGE_PATH) + + model = Model(model_config_path=CONFIG_PATH, model_checkpoint_path=WEIGHTS_PATH) + detections = model.predict_with_classes( + image=image, + classes=CLASSES, + box_threshold=BOX_THRESHOLD, + text_threshold=TEXT_THRESHOLD + ) + + + import supervision as sv + + box_annotator = sv.BoxAnnotator() + annotated_image = box_annotator.annotate(scene=image, detections=detections) + """ + caption = ". ".join(classes) + processed_image = Model.preprocess_image(image_bgr=image).to(self.device) + boxes, logits, phrases = predict( + model=self.model, + image=processed_image, + caption=caption, + box_threshold=box_threshold, + text_threshold=text_threshold, + device=self.device) + source_h, source_w, _ = image.shape + detections = Model.post_process_result( + source_h=source_h, + source_w=source_w, + boxes=boxes, + logits=logits) + class_id = Model.phrases2classes(phrases=phrases, classes=classes) + detections.class_id = class_id + return detections + + @staticmethod + def preprocess_image(image_bgr: np.ndarray) -> torch.Tensor: + transform = T.Compose( + [ + T.RandomResize([800], max_size=1333), + T.ToTensor(), + T.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]), + ] + ) + image_pillow = Image.fromarray(cv2.cvtColor(image_bgr, cv2.COLOR_BGR2RGB)) + image_transformed, _ = transform(image_pillow, None) + return image_transformed + + @staticmethod + def post_process_result( + source_h: int, + source_w: int, + boxes: torch.Tensor, + logits: torch.Tensor + ) -> sv.Detections: + boxes = boxes * torch.Tensor([source_w, source_h, source_w, source_h]) + xyxy = box_convert(boxes=boxes, in_fmt="cxcywh", out_fmt="xyxy").numpy() + confidence = logits.numpy() + return sv.Detections(xyxy=xyxy, confidence=confidence) + + @staticmethod + def phrases2classes(phrases: List[str], classes: List[str]) -> np.ndarray: + class_ids = [] + for phrase in phrases: + for class_ in classes: + if class_ in phrase: + class_ids.append(classes.index(class_)) + break + else: + class_ids.append(None) + return np.array(class_ids) diff --git a/groundingdino/util/logger.py b/groundingdino/util/logger.py new file mode 100644 index 0000000000000000000000000000000000000000..18145f54c927abd59b95f3fa6e6da8002bc2ce97 --- /dev/null +++ b/groundingdino/util/logger.py @@ -0,0 +1,93 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved +import functools +import logging +import os +import sys + +from termcolor import colored + + +class _ColorfulFormatter(logging.Formatter): + def __init__(self, *args, **kwargs): + self._root_name = kwargs.pop("root_name") + "." + self._abbrev_name = kwargs.pop("abbrev_name", "") + if len(self._abbrev_name): + self._abbrev_name = self._abbrev_name + "." + super(_ColorfulFormatter, self).__init__(*args, **kwargs) + + def formatMessage(self, record): + record.name = record.name.replace(self._root_name, self._abbrev_name) + log = super(_ColorfulFormatter, self).formatMessage(record) + if record.levelno == logging.WARNING: + prefix = colored("WARNING", "red", attrs=["blink"]) + elif record.levelno == logging.ERROR or record.levelno == logging.CRITICAL: + prefix = colored("ERROR", "red", attrs=["blink", "underline"]) + else: + return log + return prefix + " " + log + + +# so that calling setup_logger multiple times won't add many handlers +@functools.lru_cache() +def setup_logger(output=None, distributed_rank=0, *, color=True, name="imagenet", abbrev_name=None): + """ + Initialize the detectron2 logger and set its verbosity level to "INFO". + + Args: + output (str): a file name or a directory to save log. If None, will not save log file. + If ends with ".txt" or ".log", assumed to be a file name. + Otherwise, logs will be saved to `output/log.txt`. + name (str): the root module name of this logger + + Returns: + logging.Logger: a logger + """ + logger = logging.getLogger(name) + logger.setLevel(logging.DEBUG) + logger.propagate = False + + if abbrev_name is None: + abbrev_name = name + + plain_formatter = logging.Formatter( + "[%(asctime)s.%(msecs)03d]: %(message)s", datefmt="%m/%d %H:%M:%S" + ) + # stdout logging: master only + if distributed_rank == 0: + ch = logging.StreamHandler(stream=sys.stdout) + ch.setLevel(logging.DEBUG) + if color: + formatter = _ColorfulFormatter( + colored("[%(asctime)s.%(msecs)03d]: ", "green") + "%(message)s", + datefmt="%m/%d %H:%M:%S", + root_name=name, + abbrev_name=str(abbrev_name), + ) + else: + formatter = plain_formatter + ch.setFormatter(formatter) + logger.addHandler(ch) + + # file logging: all workers + if output is not None: + if output.endswith(".txt") or output.endswith(".log"): + filename = output + else: + filename = os.path.join(output, "log.txt") + if distributed_rank > 0: + filename = filename + f".rank{distributed_rank}" + os.makedirs(os.path.dirname(filename), exist_ok=True) + + fh = logging.StreamHandler(_cached_log_stream(filename)) + fh.setLevel(logging.DEBUG) + fh.setFormatter(plain_formatter) + logger.addHandler(fh) + + return logger + + +# cache the opened file object, so that different calls to `setup_logger` +# with the same file name can safely write to the same file. +@functools.lru_cache(maxsize=None) +def _cached_log_stream(filename): + return open(filename, "a") diff --git a/groundingdino/util/misc.py b/groundingdino/util/misc.py new file mode 100644 index 0000000000000000000000000000000000000000..d64b84ef24bea0c98e76824feb1903f6bfebe7a5 --- /dev/null +++ b/groundingdino/util/misc.py @@ -0,0 +1,717 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved +""" +Misc functions, including distributed helpers. + +Mostly copy-paste from torchvision references. +""" +import colorsys +import datetime +import functools +import io +import json +import os +import pickle +import subprocess +import time +from collections import OrderedDict, defaultdict, deque +from typing import List, Optional + +import numpy as np +import torch +import torch.distributed as dist + +# needed due to empty tensor bug in pytorch and torchvision 0.5 +import torchvision +from torch import Tensor + +__torchvision_need_compat_flag = float(torchvision.__version__.split(".")[1]) < 7 +if __torchvision_need_compat_flag: + from torchvision.ops import _new_empty_tensor + from torchvision.ops.misc import _output_size + + +class SmoothedValue(object): + """Track a series of values and provide access to smoothed values over a + window or the global series average. + """ + + def __init__(self, window_size=20, fmt=None): + if fmt is None: + fmt = "{median:.4f} ({global_avg:.4f})" + self.deque = deque(maxlen=window_size) + self.total = 0.0 + self.count = 0 + self.fmt = fmt + + def update(self, value, n=1): + self.deque.append(value) + self.count += n + self.total += value * n + + def synchronize_between_processes(self): + """ + Warning: does not synchronize the deque! + """ + if not is_dist_avail_and_initialized(): + return + t = torch.tensor([self.count, self.total], dtype=torch.float64, device="cuda") + dist.barrier() + dist.all_reduce(t) + t = t.tolist() + self.count = int(t[0]) + self.total = t[1] + + @property + def median(self): + d = torch.tensor(list(self.deque)) + if d.shape[0] == 0: + return 0 + return d.median().item() + + @property + def avg(self): + d = torch.tensor(list(self.deque), dtype=torch.float32) + return d.mean().item() + + @property + def global_avg(self): + if os.environ.get("SHILONG_AMP", None) == "1": + eps = 1e-4 + else: + eps = 1e-6 + return self.total / (self.count + eps) + + @property + def max(self): + return max(self.deque) + + @property + def value(self): + return self.deque[-1] + + def __str__(self): + return self.fmt.format( + median=self.median, + avg=self.avg, + global_avg=self.global_avg, + max=self.max, + value=self.value, + ) + + +@functools.lru_cache() +def _get_global_gloo_group(): + """ + Return a process group based on gloo backend, containing all the ranks + The result is cached. + """ + + if dist.get_backend() == "nccl": + return dist.new_group(backend="gloo") + + return dist.group.WORLD + + +def all_gather_cpu(data): + """ + Run all_gather on arbitrary picklable data (not necessarily tensors) + Args: + data: any picklable object + Returns: + list[data]: list of data gathered from each rank + """ + + world_size = get_world_size() + if world_size == 1: + return [data] + + cpu_group = _get_global_gloo_group() + + buffer = io.BytesIO() + torch.save(data, buffer) + data_view = buffer.getbuffer() + device = "cuda" if cpu_group is None else "cpu" + tensor = torch.ByteTensor(data_view).to(device) + + # obtain Tensor size of each rank + local_size = torch.tensor([tensor.numel()], device=device, dtype=torch.long) + size_list = [torch.tensor([0], device=device, dtype=torch.long) for _ in range(world_size)] + if cpu_group is None: + dist.all_gather(size_list, local_size) + else: + print("gathering on cpu") + dist.all_gather(size_list, local_size, group=cpu_group) + size_list = [int(size.item()) for size in size_list] + max_size = max(size_list) + assert isinstance(local_size.item(), int) + local_size = int(local_size.item()) + + # receiving Tensor from all ranks + # we pad the tensor because torch all_gather does not support + # gathering tensors of different shapes + tensor_list = [] + for _ in size_list: + tensor_list.append(torch.empty((max_size,), dtype=torch.uint8, device=device)) + if local_size != max_size: + padding = torch.empty(size=(max_size - local_size,), dtype=torch.uint8, device=device) + tensor = torch.cat((tensor, padding), dim=0) + if cpu_group is None: + dist.all_gather(tensor_list, tensor) + else: + dist.all_gather(tensor_list, tensor, group=cpu_group) + + data_list = [] + for size, tensor in zip(size_list, tensor_list): + tensor = torch.split(tensor, [size, max_size - size], dim=0)[0] + buffer = io.BytesIO(tensor.cpu().numpy()) + obj = torch.load(buffer) + data_list.append(obj) + + return data_list + + +def all_gather(data): + """ + Run all_gather on arbitrary picklable data (not necessarily tensors) + Args: + data: any picklable object + Returns: + list[data]: list of data gathered from each rank + """ + + if os.getenv("CPU_REDUCE") == "1": + return all_gather_cpu(data) + + world_size = get_world_size() + if world_size == 1: + return [data] + + # serialized to a Tensor + buffer = pickle.dumps(data) + storage = torch.ByteStorage.from_buffer(buffer) + tensor = torch.ByteTensor(storage).to("cuda") + + # obtain Tensor size of each rank + local_size = torch.tensor([tensor.numel()], device="cuda") + size_list = [torch.tensor([0], device="cuda") for _ in range(world_size)] + dist.all_gather(size_list, local_size) + size_list = [int(size.item()) for size in size_list] + max_size = max(size_list) + + # receiving Tensor from all ranks + # we pad the tensor because torch all_gather does not support + # gathering tensors of different shapes + tensor_list = [] + for _ in size_list: + tensor_list.append(torch.empty((max_size,), dtype=torch.uint8, device="cuda")) + if local_size != max_size: + padding = torch.empty(size=(max_size - local_size,), dtype=torch.uint8, device="cuda") + tensor = torch.cat((tensor, padding), dim=0) + dist.all_gather(tensor_list, tensor) + + data_list = [] + for size, tensor in zip(size_list, tensor_list): + buffer = tensor.cpu().numpy().tobytes()[:size] + data_list.append(pickle.loads(buffer)) + + return data_list + + +def reduce_dict(input_dict, average=True): + """ + Args: + input_dict (dict): all the values will be reduced + average (bool): whether to do average or sum + Reduce the values in the dictionary from all processes so that all processes + have the averaged results. Returns a dict with the same fields as + input_dict, after reduction. + """ + world_size = get_world_size() + if world_size < 2: + return input_dict + with torch.no_grad(): + names = [] + values = [] + # sort the keys so that they are consistent across processes + for k in sorted(input_dict.keys()): + names.append(k) + values.append(input_dict[k]) + values = torch.stack(values, dim=0) + dist.all_reduce(values) + if average: + values /= world_size + reduced_dict = {k: v for k, v in zip(names, values)} + return reduced_dict + + +class MetricLogger(object): + def __init__(self, delimiter="\t"): + self.meters = defaultdict(SmoothedValue) + self.delimiter = delimiter + + def update(self, **kwargs): + for k, v in kwargs.items(): + if isinstance(v, torch.Tensor): + v = v.item() + assert isinstance(v, (float, int)) + self.meters[k].update(v) + + def __getattr__(self, attr): + if attr in self.meters: + return self.meters[attr] + if attr in self.__dict__: + return self.__dict__[attr] + raise AttributeError("'{}' object has no attribute '{}'".format(type(self).__name__, attr)) + + def __str__(self): + loss_str = [] + for name, meter in self.meters.items(): + # print(name, str(meter)) + # import ipdb;ipdb.set_trace() + if meter.count > 0: + loss_str.append("{}: {}".format(name, str(meter))) + return self.delimiter.join(loss_str) + + def synchronize_between_processes(self): + for meter in self.meters.values(): + meter.synchronize_between_processes() + + def add_meter(self, name, meter): + self.meters[name] = meter + + def log_every(self, iterable, print_freq, header=None, logger=None): + if logger is None: + print_func = print + else: + print_func = logger.info + + i = 0 + if not header: + header = "" + start_time = time.time() + end = time.time() + iter_time = SmoothedValue(fmt="{avg:.4f}") + data_time = SmoothedValue(fmt="{avg:.4f}") + space_fmt = ":" + str(len(str(len(iterable)))) + "d" + if torch.cuda.is_available(): + log_msg = self.delimiter.join( + [ + header, + "[{0" + space_fmt + "}/{1}]", + "eta: {eta}", + "{meters}", + "time: {time}", + "data: {data}", + "max mem: {memory:.0f}", + ] + ) + else: + log_msg = self.delimiter.join( + [ + header, + "[{0" + space_fmt + "}/{1}]", + "eta: {eta}", + "{meters}", + "time: {time}", + "data: {data}", + ] + ) + MB = 1024.0 * 1024.0 + for obj in iterable: + data_time.update(time.time() - end) + yield obj + # import ipdb; ipdb.set_trace() + iter_time.update(time.time() - end) + if i % print_freq == 0 or i == len(iterable) - 1: + eta_seconds = iter_time.global_avg * (len(iterable) - i) + eta_string = str(datetime.timedelta(seconds=int(eta_seconds))) + if torch.cuda.is_available(): + print_func( + log_msg.format( + i, + len(iterable), + eta=eta_string, + meters=str(self), + time=str(iter_time), + data=str(data_time), + memory=torch.cuda.max_memory_allocated() / MB, + ) + ) + else: + print_func( + log_msg.format( + i, + len(iterable), + eta=eta_string, + meters=str(self), + time=str(iter_time), + data=str(data_time), + ) + ) + i += 1 + end = time.time() + total_time = time.time() - start_time + total_time_str = str(datetime.timedelta(seconds=int(total_time))) + print_func( + "{} Total time: {} ({:.4f} s / it)".format( + header, total_time_str, total_time / len(iterable) + ) + ) + + +def get_sha(): + cwd = os.path.dirname(os.path.abspath(__file__)) + + def _run(command): + return subprocess.check_output(command, cwd=cwd).decode("ascii").strip() + + sha = "N/A" + diff = "clean" + branch = "N/A" + try: + sha = _run(["git", "rev-parse", "HEAD"]) + subprocess.check_output(["git", "diff"], cwd=cwd) + diff = _run(["git", "diff-index", "HEAD"]) + diff = "has uncommited changes" if diff else "clean" + branch = _run(["git", "rev-parse", "--abbrev-ref", "HEAD"]) + except Exception: + pass + message = f"sha: {sha}, status: {diff}, branch: {branch}" + return message + + +def collate_fn(batch): + # import ipdb; ipdb.set_trace() + batch = list(zip(*batch)) + batch[0] = nested_tensor_from_tensor_list(batch[0]) + return tuple(batch) + + +def _max_by_axis(the_list): + # type: (List[List[int]]) -> List[int] + maxes = the_list[0] + for sublist in the_list[1:]: + for index, item in enumerate(sublist): + maxes[index] = max(maxes[index], item) + return maxes + + +class NestedTensor(object): + def __init__(self, tensors, mask: Optional[Tensor]): + self.tensors = tensors + self.mask = mask + if mask == "auto": + self.mask = torch.zeros_like(tensors).to(tensors.device) + if self.mask.dim() == 3: + self.mask = self.mask.sum(0).to(bool) + elif self.mask.dim() == 4: + self.mask = self.mask.sum(1).to(bool) + else: + raise ValueError( + "tensors dim must be 3 or 4 but {}({})".format( + self.tensors.dim(), self.tensors.shape + ) + ) + + def imgsize(self): + res = [] + for i in range(self.tensors.shape[0]): + mask = self.mask[i] + maxH = (~mask).sum(0).max() + maxW = (~mask).sum(1).max() + res.append(torch.Tensor([maxH, maxW])) + return res + + def to(self, device): + # type: (Device) -> NestedTensor # noqa + cast_tensor = self.tensors.to(device) + mask = self.mask + if mask is not None: + assert mask is not None + cast_mask = mask.to(device) + else: + cast_mask = None + return NestedTensor(cast_tensor, cast_mask) + + def to_img_list_single(self, tensor, mask): + assert tensor.dim() == 3, "dim of tensor should be 3 but {}".format(tensor.dim()) + maxH = (~mask).sum(0).max() + maxW = (~mask).sum(1).max() + img = tensor[:, :maxH, :maxW] + return img + + def to_img_list(self): + """remove the padding and convert to img list + + Returns: + [type]: [description] + """ + if self.tensors.dim() == 3: + return self.to_img_list_single(self.tensors, self.mask) + else: + res = [] + for i in range(self.tensors.shape[0]): + tensor_i = self.tensors[i] + mask_i = self.mask[i] + res.append(self.to_img_list_single(tensor_i, mask_i)) + return res + + @property + def device(self): + return self.tensors.device + + def decompose(self): + return self.tensors, self.mask + + def __repr__(self): + return str(self.tensors) + + @property + def shape(self): + return {"tensors.shape": self.tensors.shape, "mask.shape": self.mask.shape} + + +def nested_tensor_from_tensor_list(tensor_list: List[Tensor]): + # TODO make this more general + if tensor_list[0].ndim == 3: + if torchvision._is_tracing(): + # nested_tensor_from_tensor_list() does not export well to ONNX + # call _onnx_nested_tensor_from_tensor_list() instead + return _onnx_nested_tensor_from_tensor_list(tensor_list) + + # TODO make it support different-sized images + max_size = _max_by_axis([list(img.shape) for img in tensor_list]) + # min_size = tuple(min(s) for s in zip(*[img.shape for img in tensor_list])) + batch_shape = [len(tensor_list)] + max_size + b, c, h, w = batch_shape + dtype = tensor_list[0].dtype + device = tensor_list[0].device + tensor = torch.zeros(batch_shape, dtype=dtype, device=device) + mask = torch.ones((b, h, w), dtype=torch.bool, device=device) + for img, pad_img, m in zip(tensor_list, tensor, mask): + pad_img[: img.shape[0], : img.shape[1], : img.shape[2]].copy_(img) + m[: img.shape[1], : img.shape[2]] = False + else: + raise ValueError("not supported") + return NestedTensor(tensor, mask) + + +# _onnx_nested_tensor_from_tensor_list() is an implementation of +# nested_tensor_from_tensor_list() that is supported by ONNX tracing. +@torch.jit.unused +def _onnx_nested_tensor_from_tensor_list(tensor_list: List[Tensor]) -> NestedTensor: + max_size = [] + for i in range(tensor_list[0].dim()): + max_size_i = torch.max( + torch.stack([img.shape[i] for img in tensor_list]).to(torch.float32) + ).to(torch.int64) + max_size.append(max_size_i) + max_size = tuple(max_size) + + # work around for + # pad_img[: img.shape[0], : img.shape[1], : img.shape[2]].copy_(img) + # m[: img.shape[1], :img.shape[2]] = False + # which is not yet supported in onnx + padded_imgs = [] + padded_masks = [] + for img in tensor_list: + padding = [(s1 - s2) for s1, s2 in zip(max_size, tuple(img.shape))] + padded_img = torch.nn.functional.pad(img, (0, padding[2], 0, padding[1], 0, padding[0])) + padded_imgs.append(padded_img) + + m = torch.zeros_like(img[0], dtype=torch.int, device=img.device) + padded_mask = torch.nn.functional.pad(m, (0, padding[2], 0, padding[1]), "constant", 1) + padded_masks.append(padded_mask.to(torch.bool)) + + tensor = torch.stack(padded_imgs) + mask = torch.stack(padded_masks) + + return NestedTensor(tensor, mask=mask) + + +def setup_for_distributed(is_master): + """ + This function disables printing when not in master process + """ + import builtins as __builtin__ + + builtin_print = __builtin__.print + + def print(*args, **kwargs): + force = kwargs.pop("force", False) + if is_master or force: + builtin_print(*args, **kwargs) + + __builtin__.print = print + + +def is_dist_avail_and_initialized(): + if not dist.is_available(): + return False + if not dist.is_initialized(): + return False + return True + + +def get_world_size(): + if not is_dist_avail_and_initialized(): + return 1 + return dist.get_world_size() + + +def get_rank(): + if not is_dist_avail_and_initialized(): + return 0 + return dist.get_rank() + + +def is_main_process(): + return get_rank() == 0 + + +def save_on_master(*args, **kwargs): + if is_main_process(): + torch.save(*args, **kwargs) + + +def init_distributed_mode(args): + if "WORLD_SIZE" in os.environ and os.environ["WORLD_SIZE"] != "": # 'RANK' in os.environ and + args.rank = int(os.environ["RANK"]) + args.world_size = int(os.environ["WORLD_SIZE"]) + args.gpu = args.local_rank = int(os.environ["LOCAL_RANK"]) + + # launch by torch.distributed.launch + # Single node + # python -m torch.distributed.launch --nproc_per_node=8 main.py --world-size 1 --rank 0 ... + # Multi nodes + # python -m torch.distributed.launch --nproc_per_node=8 main.py --world-size 2 --rank 0 --dist-url 'tcp://IP_OF_NODE0:FREEPORT' ... + # python -m torch.distributed.launch --nproc_per_node=8 main.py --world-size 2 --rank 1 --dist-url 'tcp://IP_OF_NODE0:FREEPORT' ... + # args.rank = int(os.environ.get('OMPI_COMM_WORLD_RANK')) + # local_world_size = int(os.environ['GPU_PER_NODE_COUNT']) + # args.world_size = args.world_size * local_world_size + # args.gpu = args.local_rank = int(os.environ['LOCAL_RANK']) + # args.rank = args.rank * local_world_size + args.local_rank + print( + "world size: {}, rank: {}, local rank: {}".format( + args.world_size, args.rank, args.local_rank + ) + ) + print(json.dumps(dict(os.environ), indent=2)) + elif "SLURM_PROCID" in os.environ: + args.rank = int(os.environ["SLURM_PROCID"]) + args.gpu = args.local_rank = int(os.environ["SLURM_LOCALID"]) + args.world_size = int(os.environ["SLURM_NPROCS"]) + + print( + "world size: {}, world rank: {}, local rank: {}, device_count: {}".format( + args.world_size, args.rank, args.local_rank, torch.cuda.device_count() + ) + ) + else: + print("Not using distributed mode") + args.distributed = False + args.world_size = 1 + args.rank = 0 + args.local_rank = 0 + return + + print("world_size:{} rank:{} local_rank:{}".format(args.world_size, args.rank, args.local_rank)) + args.distributed = True + torch.cuda.set_device(args.local_rank) + args.dist_backend = "nccl" + print("| distributed init (rank {}): {}".format(args.rank, args.dist_url), flush=True) + + torch.distributed.init_process_group( + backend=args.dist_backend, + world_size=args.world_size, + rank=args.rank, + init_method=args.dist_url, + ) + + print("Before torch.distributed.barrier()") + torch.distributed.barrier() + print("End torch.distributed.barrier()") + setup_for_distributed(args.rank == 0) + + +@torch.no_grad() +def accuracy(output, target, topk=(1,)): + """Computes the precision@k for the specified values of k""" + if target.numel() == 0: + return [torch.zeros([], device=output.device)] + maxk = max(topk) + batch_size = target.size(0) + + _, pred = output.topk(maxk, 1, True, True) + pred = pred.t() + correct = pred.eq(target.view(1, -1).expand_as(pred)) + + res = [] + for k in topk: + correct_k = correct[:k].view(-1).float().sum(0) + res.append(correct_k.mul_(100.0 / batch_size)) + return res + + +@torch.no_grad() +def accuracy_onehot(pred, gt): + """_summary_ + + Args: + pred (_type_): n, c + gt (_type_): n, c + """ + tp = ((pred - gt).abs().sum(-1) < 1e-4).float().sum() + acc = tp / gt.shape[0] * 100 + return acc + + +def interpolate(input, size=None, scale_factor=None, mode="nearest", align_corners=None): + # type: (Tensor, Optional[List[int]], Optional[float], str, Optional[bool]) -> Tensor + """ + Equivalent to nn.functional.interpolate, but with support for empty batch sizes. + This will eventually be supported natively by PyTorch, and this + class can go away. + """ + if __torchvision_need_compat_flag < 0.7: + if input.numel() > 0: + return torch.nn.functional.interpolate(input, size, scale_factor, mode, align_corners) + + output_shape = _output_size(2, input, size, scale_factor) + output_shape = list(input.shape[:-2]) + list(output_shape) + return _new_empty_tensor(input, output_shape) + else: + return torchvision.ops.misc.interpolate(input, size, scale_factor, mode, align_corners) + + +class color_sys: + def __init__(self, num_colors) -> None: + self.num_colors = num_colors + colors = [] + for i in np.arange(0.0, 360.0, 360.0 / num_colors): + hue = i / 360.0 + lightness = (50 + np.random.rand() * 10) / 100.0 + saturation = (90 + np.random.rand() * 10) / 100.0 + colors.append( + tuple([int(j * 255) for j in colorsys.hls_to_rgb(hue, lightness, saturation)]) + ) + self.colors = colors + + def __call__(self, idx): + return self.colors[idx] + + +def inverse_sigmoid(x, eps=1e-3): + x = x.clamp(min=0, max=1) + x1 = x.clamp(min=eps) + x2 = (1 - x).clamp(min=eps) + return torch.log(x1 / x2) + + +def clean_state_dict(state_dict): + new_state_dict = OrderedDict() + for k, v in state_dict.items(): + if k[:7] == "module.": + k = k[7:] # remove `module.` + new_state_dict[k] = v + return new_state_dict diff --git a/groundingdino/util/slconfig.py b/groundingdino/util/slconfig.py new file mode 100644 index 0000000000000000000000000000000000000000..672e72ed0b68a54c13ade66c9f146d2d542e97c6 --- /dev/null +++ b/groundingdino/util/slconfig.py @@ -0,0 +1,427 @@ +# ========================================================== +# Modified from mmcv +# ========================================================== +import ast +import os +import os.path as osp +import shutil +import sys +import tempfile +from argparse import Action +from importlib import import_module + +from addict import Dict +from yapf.yapflib.yapf_api import FormatCode + +BASE_KEY = "_base_" +DELETE_KEY = "_delete_" +RESERVED_KEYS = ["filename", "text", "pretty_text", "get", "dump", "merge_from_dict"] + + +def check_file_exist(filename, msg_tmpl='file "{}" does not exist'): + if not osp.isfile(filename): + raise FileNotFoundError(msg_tmpl.format(filename)) + + +class ConfigDict(Dict): + def __missing__(self, name): + raise KeyError(name) + + def __getattr__(self, name): + try: + value = super(ConfigDict, self).__getattr__(name) + except KeyError: + ex = AttributeError(f"'{self.__class__.__name__}' object has no " f"attribute '{name}'") + except Exception as e: + ex = e + else: + return value + raise ex + + +class SLConfig(object): + """ + config files. + only support .py file as config now. + + ref: mmcv.utils.config + + Example: + >>> cfg = Config(dict(a=1, b=dict(b1=[0, 1]))) + >>> cfg.a + 1 + >>> cfg.b + {'b1': [0, 1]} + >>> cfg.b.b1 + [0, 1] + >>> cfg = Config.fromfile('tests/data/config/a.py') + >>> cfg.filename + "/home/kchen/projects/mmcv/tests/data/config/a.py" + >>> cfg.item4 + 'test' + >>> cfg + "Config [path: /home/kchen/projects/mmcv/tests/data/config/a.py]: " + "{'item1': [1, 2], 'item2': {'a': 0}, 'item3': True, 'item4': 'test'}" + """ + + @staticmethod + def _validate_py_syntax(filename): + with open(filename) as f: + content = f.read() + try: + ast.parse(content) + except SyntaxError: + raise SyntaxError("There are syntax errors in config " f"file {filename}") + + @staticmethod + def _file2dict(filename): + filename = osp.abspath(osp.expanduser(filename)) + check_file_exist(filename) + if filename.lower().endswith(".py"): + with tempfile.TemporaryDirectory() as temp_config_dir: + temp_config_file = tempfile.NamedTemporaryFile(dir=temp_config_dir, suffix=".py") + temp_config_name = osp.basename(temp_config_file.name) + if os.name == 'nt': + temp_config_file.close() + shutil.copyfile(filename, osp.join(temp_config_dir, temp_config_name)) + temp_module_name = osp.splitext(temp_config_name)[0] + sys.path.insert(0, temp_config_dir) + SLConfig._validate_py_syntax(filename) + mod = import_module(temp_module_name) + sys.path.pop(0) + cfg_dict = { + name: value for name, value in mod.__dict__.items() if not name.startswith("__") + } + # delete imported module + del sys.modules[temp_module_name] + # close temp file + temp_config_file.close() + elif filename.lower().endswith((".yml", ".yaml", ".json")): + from .slio import slload + + cfg_dict = slload(filename) + else: + raise IOError("Only py/yml/yaml/json type are supported now!") + + cfg_text = filename + "\n" + with open(filename, "r") as f: + cfg_text += f.read() + + # parse the base file + if BASE_KEY in cfg_dict: + cfg_dir = osp.dirname(filename) + base_filename = cfg_dict.pop(BASE_KEY) + base_filename = base_filename if isinstance(base_filename, list) else [base_filename] + + cfg_dict_list = list() + cfg_text_list = list() + for f in base_filename: + _cfg_dict, _cfg_text = SLConfig._file2dict(osp.join(cfg_dir, f)) + cfg_dict_list.append(_cfg_dict) + cfg_text_list.append(_cfg_text) + + base_cfg_dict = dict() + for c in cfg_dict_list: + if len(base_cfg_dict.keys() & c.keys()) > 0: + raise KeyError("Duplicate key is not allowed among bases") + # TODO Allow the duplicate key while warnning user + base_cfg_dict.update(c) + + base_cfg_dict = SLConfig._merge_a_into_b(cfg_dict, base_cfg_dict) + cfg_dict = base_cfg_dict + + # merge cfg_text + cfg_text_list.append(cfg_text) + cfg_text = "\n".join(cfg_text_list) + + return cfg_dict, cfg_text + + @staticmethod + def _merge_a_into_b(a, b): + """merge dict `a` into dict `b` (non-inplace). + values in `a` will overwrite `b`. + copy first to avoid inplace modification + + Args: + a ([type]): [description] + b ([type]): [description] + + Returns: + [dict]: [description] + """ + # import ipdb; ipdb.set_trace() + if not isinstance(a, dict): + return a + + b = b.copy() + for k, v in a.items(): + if isinstance(v, dict) and k in b and not v.pop(DELETE_KEY, False): + + if not isinstance(b[k], dict) and not isinstance(b[k], list): + # if : + # import ipdb; ipdb.set_trace() + raise TypeError( + f"{k}={v} in child config cannot inherit from base " + f"because {k} is a dict in the child config but is of " + f"type {type(b[k])} in base config. You may set " + f"`{DELETE_KEY}=True` to ignore the base config" + ) + b[k] = SLConfig._merge_a_into_b(v, b[k]) + elif isinstance(b, list): + try: + _ = int(k) + except: + raise TypeError( + f"b is a list, " f"index {k} should be an int when input but {type(k)}" + ) + b[int(k)] = SLConfig._merge_a_into_b(v, b[int(k)]) + else: + b[k] = v + + return b + + @staticmethod + def fromfile(filename): + cfg_dict, cfg_text = SLConfig._file2dict(filename) + return SLConfig(cfg_dict, cfg_text=cfg_text, filename=filename) + + def __init__(self, cfg_dict=None, cfg_text=None, filename=None): + if cfg_dict is None: + cfg_dict = dict() + elif not isinstance(cfg_dict, dict): + raise TypeError("cfg_dict must be a dict, but " f"got {type(cfg_dict)}") + for key in cfg_dict: + if key in RESERVED_KEYS: + raise KeyError(f"{key} is reserved for config file") + + super(SLConfig, self).__setattr__("_cfg_dict", ConfigDict(cfg_dict)) + super(SLConfig, self).__setattr__("_filename", filename) + if cfg_text: + text = cfg_text + elif filename: + with open(filename, "r") as f: + text = f.read() + else: + text = "" + super(SLConfig, self).__setattr__("_text", text) + + @property + def filename(self): + return self._filename + + @property + def text(self): + return self._text + + @property + def pretty_text(self): + + indent = 4 + + def _indent(s_, num_spaces): + s = s_.split("\n") + if len(s) == 1: + return s_ + first = s.pop(0) + s = [(num_spaces * " ") + line for line in s] + s = "\n".join(s) + s = first + "\n" + s + return s + + def _format_basic_types(k, v, use_mapping=False): + if isinstance(v, str): + v_str = f"'{v}'" + else: + v_str = str(v) + + if use_mapping: + k_str = f"'{k}'" if isinstance(k, str) else str(k) + attr_str = f"{k_str}: {v_str}" + else: + attr_str = f"{str(k)}={v_str}" + attr_str = _indent(attr_str, indent) + + return attr_str + + def _format_list(k, v, use_mapping=False): + # check if all items in the list are dict + if all(isinstance(_, dict) for _ in v): + v_str = "[\n" + v_str += "\n".join( + f"dict({_indent(_format_dict(v_), indent)})," for v_ in v + ).rstrip(",") + if use_mapping: + k_str = f"'{k}'" if isinstance(k, str) else str(k) + attr_str = f"{k_str}: {v_str}" + else: + attr_str = f"{str(k)}={v_str}" + attr_str = _indent(attr_str, indent) + "]" + else: + attr_str = _format_basic_types(k, v, use_mapping) + return attr_str + + def _contain_invalid_identifier(dict_str): + contain_invalid_identifier = False + for key_name in dict_str: + contain_invalid_identifier |= not str(key_name).isidentifier() + return contain_invalid_identifier + + def _format_dict(input_dict, outest_level=False): + r = "" + s = [] + + use_mapping = _contain_invalid_identifier(input_dict) + if use_mapping: + r += "{" + for idx, (k, v) in enumerate(input_dict.items()): + is_last = idx >= len(input_dict) - 1 + end = "" if outest_level or is_last else "," + if isinstance(v, dict): + v_str = "\n" + _format_dict(v) + if use_mapping: + k_str = f"'{k}'" if isinstance(k, str) else str(k) + attr_str = f"{k_str}: dict({v_str}" + else: + attr_str = f"{str(k)}=dict({v_str}" + attr_str = _indent(attr_str, indent) + ")" + end + elif isinstance(v, list): + attr_str = _format_list(k, v, use_mapping) + end + else: + attr_str = _format_basic_types(k, v, use_mapping) + end + + s.append(attr_str) + r += "\n".join(s) + if use_mapping: + r += "}" + return r + + cfg_dict = self._cfg_dict.to_dict() + text = _format_dict(cfg_dict, outest_level=True) + # copied from setup.cfg + yapf_style = dict( + based_on_style="pep8", + blank_line_before_nested_class_or_def=True, + split_before_expression_after_opening_paren=True, + ) + text, _ = FormatCode(text, style_config=yapf_style, verify=True) + + return text + + def __repr__(self): + return f"Config (path: {self.filename}): {self._cfg_dict.__repr__()}" + + def __len__(self): + return len(self._cfg_dict) + + def __getattr__(self, name): + # # debug + # print('+'*15) + # print('name=%s' % name) + # print("addr:", id(self)) + # # print('type(self):', type(self)) + # print(self.__dict__) + # print('+'*15) + # if self.__dict__ == {}: + # raise ValueError + + return getattr(self._cfg_dict, name) + + def __getitem__(self, name): + return self._cfg_dict.__getitem__(name) + + def __setattr__(self, name, value): + if isinstance(value, dict): + value = ConfigDict(value) + self._cfg_dict.__setattr__(name, value) + + def __setitem__(self, name, value): + if isinstance(value, dict): + value = ConfigDict(value) + self._cfg_dict.__setitem__(name, value) + + def __iter__(self): + return iter(self._cfg_dict) + + def dump(self, file=None): + # import ipdb; ipdb.set_trace() + if file is None: + return self.pretty_text + else: + with open(file, "w") as f: + f.write(self.pretty_text) + + def merge_from_dict(self, options): + """Merge list into cfg_dict + + Merge the dict parsed by MultipleKVAction into this cfg. + + Examples: + >>> options = {'model.backbone.depth': 50, + ... 'model.backbone.with_cp':True} + >>> cfg = Config(dict(model=dict(backbone=dict(type='ResNet')))) + >>> cfg.merge_from_dict(options) + >>> cfg_dict = super(Config, self).__getattribute__('_cfg_dict') + >>> assert cfg_dict == dict( + ... model=dict(backbone=dict(depth=50, with_cp=True))) + + Args: + options (dict): dict of configs to merge from. + """ + option_cfg_dict = {} + for full_key, v in options.items(): + d = option_cfg_dict + key_list = full_key.split(".") + for subkey in key_list[:-1]: + d.setdefault(subkey, ConfigDict()) + d = d[subkey] + subkey = key_list[-1] + d[subkey] = v + + cfg_dict = super(SLConfig, self).__getattribute__("_cfg_dict") + super(SLConfig, self).__setattr__( + "_cfg_dict", SLConfig._merge_a_into_b(option_cfg_dict, cfg_dict) + ) + + # for multiprocess + def __setstate__(self, state): + self.__init__(state) + + def copy(self): + return SLConfig(self._cfg_dict.copy()) + + def deepcopy(self): + return SLConfig(self._cfg_dict.deepcopy()) + + +class DictAction(Action): + """ + argparse action to split an argument into KEY=VALUE form + on the first = and append to a dictionary. List options should + be passed as comma separated values, i.e KEY=V1,V2,V3 + """ + + @staticmethod + def _parse_int_float_bool(val): + try: + return int(val) + except ValueError: + pass + try: + return float(val) + except ValueError: + pass + if val.lower() in ["true", "false"]: + return True if val.lower() == "true" else False + if val.lower() in ["none", "null"]: + return None + return val + + def __call__(self, parser, namespace, values, option_string=None): + options = {} + for kv in values: + key, val = kv.split("=", maxsplit=1) + val = [self._parse_int_float_bool(v) for v in val.split(",")] + if len(val) == 1: + val = val[0] + options[key] = val + setattr(namespace, self.dest, options) diff --git a/groundingdino/util/slio.py b/groundingdino/util/slio.py new file mode 100644 index 0000000000000000000000000000000000000000..72c1f0f7b82cdc931d381feef64fe15815ba657e --- /dev/null +++ b/groundingdino/util/slio.py @@ -0,0 +1,177 @@ +# ========================================================== +# Modified from mmcv +# ========================================================== + +import json +import pickle +from abc import ABCMeta, abstractmethod +from pathlib import Path + +import yaml + +try: + from yaml import CLoader as Loader, CDumper as Dumper +except ImportError: + from yaml import Loader, Dumper + + +# =========================== +# Rigister handler +# =========================== + + +class BaseFileHandler(metaclass=ABCMeta): + @abstractmethod + def load_from_fileobj(self, file, **kwargs): + pass + + @abstractmethod + def dump_to_fileobj(self, obj, file, **kwargs): + pass + + @abstractmethod + def dump_to_str(self, obj, **kwargs): + pass + + def load_from_path(self, filepath, mode="r", **kwargs): + with open(filepath, mode) as f: + return self.load_from_fileobj(f, **kwargs) + + def dump_to_path(self, obj, filepath, mode="w", **kwargs): + with open(filepath, mode) as f: + self.dump_to_fileobj(obj, f, **kwargs) + + +class JsonHandler(BaseFileHandler): + def load_from_fileobj(self, file): + return json.load(file) + + def dump_to_fileobj(self, obj, file, **kwargs): + json.dump(obj, file, **kwargs) + + def dump_to_str(self, obj, **kwargs): + return json.dumps(obj, **kwargs) + + +class PickleHandler(BaseFileHandler): + def load_from_fileobj(self, file, **kwargs): + return pickle.load(file, **kwargs) + + def load_from_path(self, filepath, **kwargs): + return super(PickleHandler, self).load_from_path(filepath, mode="rb", **kwargs) + + def dump_to_str(self, obj, **kwargs): + kwargs.setdefault("protocol", 2) + return pickle.dumps(obj, **kwargs) + + def dump_to_fileobj(self, obj, file, **kwargs): + kwargs.setdefault("protocol", 2) + pickle.dump(obj, file, **kwargs) + + def dump_to_path(self, obj, filepath, **kwargs): + super(PickleHandler, self).dump_to_path(obj, filepath, mode="wb", **kwargs) + + +class YamlHandler(BaseFileHandler): + def load_from_fileobj(self, file, **kwargs): + kwargs.setdefault("Loader", Loader) + return yaml.load(file, **kwargs) + + def dump_to_fileobj(self, obj, file, **kwargs): + kwargs.setdefault("Dumper", Dumper) + yaml.dump(obj, file, **kwargs) + + def dump_to_str(self, obj, **kwargs): + kwargs.setdefault("Dumper", Dumper) + return yaml.dump(obj, **kwargs) + + +file_handlers = { + "json": JsonHandler(), + "yaml": YamlHandler(), + "yml": YamlHandler(), + "pickle": PickleHandler(), + "pkl": PickleHandler(), +} + +# =========================== +# load and dump +# =========================== + + +def is_str(x): + """Whether the input is an string instance. + + Note: This method is deprecated since python 2 is no longer supported. + """ + return isinstance(x, str) + + +def slload(file, file_format=None, **kwargs): + """Load data from json/yaml/pickle files. + + This method provides a unified api for loading data from serialized files. + + Args: + file (str or :obj:`Path` or file-like object): Filename or a file-like + object. + file_format (str, optional): If not specified, the file format will be + inferred from the file extension, otherwise use the specified one. + Currently supported formats include "json", "yaml/yml" and + "pickle/pkl". + + Returns: + The content from the file. + """ + if isinstance(file, Path): + file = str(file) + if file_format is None and is_str(file): + file_format = file.split(".")[-1] + if file_format not in file_handlers: + raise TypeError(f"Unsupported format: {file_format}") + + handler = file_handlers[file_format] + if is_str(file): + obj = handler.load_from_path(file, **kwargs) + elif hasattr(file, "read"): + obj = handler.load_from_fileobj(file, **kwargs) + else: + raise TypeError('"file" must be a filepath str or a file-object') + return obj + + +def sldump(obj, file=None, file_format=None, **kwargs): + """Dump data to json/yaml/pickle strings or files. + + This method provides a unified api for dumping data as strings or to files, + and also supports custom arguments for each file format. + + Args: + obj (any): The python object to be dumped. + file (str or :obj:`Path` or file-like object, optional): If not + specified, then the object is dump to a str, otherwise to a file + specified by the filename or file-like object. + file_format (str, optional): Same as :func:`load`. + + Returns: + bool: True for success, False otherwise. + """ + if isinstance(file, Path): + file = str(file) + if file_format is None: + if is_str(file): + file_format = file.split(".")[-1] + elif file is None: + raise ValueError("file_format must be specified since file is None") + if file_format not in file_handlers: + raise TypeError(f"Unsupported format: {file_format}") + + handler = file_handlers[file_format] + if file is None: + return handler.dump_to_str(obj, **kwargs) + elif is_str(file): + handler.dump_to_path(obj, file, **kwargs) + elif hasattr(file, "write"): + handler.dump_to_fileobj(obj, file, **kwargs) + else: + raise TypeError('"file" must be a filename str or a file-object') diff --git a/groundingdino/util/time_counter.py b/groundingdino/util/time_counter.py new file mode 100644 index 0000000000000000000000000000000000000000..0aedb2e4d61bfbe7571dca9d50053f0fedaa1359 --- /dev/null +++ b/groundingdino/util/time_counter.py @@ -0,0 +1,62 @@ +import json +import time + + +class TimeCounter: + def __init__(self) -> None: + pass + + def clear(self): + self.timedict = {} + self.basetime = time.perf_counter() + + def timeit(self, name): + nowtime = time.perf_counter() - self.basetime + self.timedict[name] = nowtime + self.basetime = time.perf_counter() + + +class TimeHolder: + def __init__(self) -> None: + self.timedict = {} + + def update(self, _timedict: dict): + for k, v in _timedict.items(): + if k not in self.timedict: + self.timedict[k] = AverageMeter(name=k, val_only=True) + self.timedict[k].update(val=v) + + def final_res(self): + return {k: v.avg for k, v in self.timedict.items()} + + def __str__(self): + return json.dumps(self.final_res(), indent=2) + + +class AverageMeter(object): + """Computes and stores the average and current value""" + + def __init__(self, name, fmt=":f", val_only=False): + self.name = name + self.fmt = fmt + self.val_only = val_only + self.reset() + + def reset(self): + self.val = 0 + self.avg = 0 + self.sum = 0 + self.count = 0 + + def update(self, val, n=1): + self.val = val + self.sum += val * n + self.count += n + self.avg = self.sum / self.count + + def __str__(self): + if self.val_only: + fmtstr = "{name} {val" + self.fmt + "}" + else: + fmtstr = "{name} {val" + self.fmt + "} ({avg" + self.fmt + "})" + return fmtstr.format(**self.__dict__) diff --git a/groundingdino/util/utils.py b/groundingdino/util/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..8cf83ae03a7865bc48493be16e8b1b2d53a1b09f --- /dev/null +++ b/groundingdino/util/utils.py @@ -0,0 +1,610 @@ +import argparse +import json +import warnings +from collections import OrderedDict +from copy import deepcopy +from typing import Any, Dict, List + +import numpy as np +import torch +from transformers import AutoTokenizer + +from groundingdino.util.slconfig import SLConfig + + +def slprint(x, name="x"): + if isinstance(x, (torch.Tensor, np.ndarray)): + print(f"{name}.shape:", x.shape) + elif isinstance(x, (tuple, list)): + print("type x:", type(x)) + for i in range(min(10, len(x))): + slprint(x[i], f"{name}[{i}]") + elif isinstance(x, dict): + for k, v in x.items(): + slprint(v, f"{name}[{k}]") + else: + print(f"{name}.type:", type(x)) + + +def clean_state_dict(state_dict): + new_state_dict = OrderedDict() + for k, v in state_dict.items(): + if k[:7] == "module.": + k = k[7:] # remove `module.` + new_state_dict[k] = v + return new_state_dict + + +def renorm( + img: torch.FloatTensor, mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225] +) -> torch.FloatTensor: + # img: tensor(3,H,W) or tensor(B,3,H,W) + # return: same as img + assert img.dim() == 3 or img.dim() == 4, "img.dim() should be 3 or 4 but %d" % img.dim() + if img.dim() == 3: + assert img.size(0) == 3, 'img.size(0) shoule be 3 but "%d". (%s)' % ( + img.size(0), + str(img.size()), + ) + img_perm = img.permute(1, 2, 0) + mean = torch.Tensor(mean) + std = torch.Tensor(std) + img_res = img_perm * std + mean + return img_res.permute(2, 0, 1) + else: # img.dim() == 4 + assert img.size(1) == 3, 'img.size(1) shoule be 3 but "%d". (%s)' % ( + img.size(1), + str(img.size()), + ) + img_perm = img.permute(0, 2, 3, 1) + mean = torch.Tensor(mean) + std = torch.Tensor(std) + img_res = img_perm * std + mean + return img_res.permute(0, 3, 1, 2) + + +class CocoClassMapper: + def __init__(self) -> None: + self.category_map_str = { + "1": 1, + "2": 2, + "3": 3, + "4": 4, + "5": 5, + "6": 6, + "7": 7, + "8": 8, + "9": 9, + "10": 10, + "11": 11, + "13": 12, + "14": 13, + "15": 14, + "16": 15, + "17": 16, + "18": 17, + "19": 18, + "20": 19, + "21": 20, + "22": 21, + "23": 22, + "24": 23, + "25": 24, + "27": 25, + "28": 26, + "31": 27, + "32": 28, + "33": 29, + "34": 30, + "35": 31, + "36": 32, + "37": 33, + "38": 34, + "39": 35, + "40": 36, + "41": 37, + "42": 38, + "43": 39, + "44": 40, + "46": 41, + "47": 42, + "48": 43, + "49": 44, + "50": 45, + "51": 46, + "52": 47, + "53": 48, + "54": 49, + "55": 50, + "56": 51, + "57": 52, + "58": 53, + "59": 54, + "60": 55, + "61": 56, + "62": 57, + "63": 58, + "64": 59, + "65": 60, + "67": 61, + "70": 62, + "72": 63, + "73": 64, + "74": 65, + "75": 66, + "76": 67, + "77": 68, + "78": 69, + "79": 70, + "80": 71, + "81": 72, + "82": 73, + "84": 74, + "85": 75, + "86": 76, + "87": 77, + "88": 78, + "89": 79, + "90": 80, + } + self.origin2compact_mapper = {int(k): v - 1 for k, v in self.category_map_str.items()} + self.compact2origin_mapper = {int(v - 1): int(k) for k, v in self.category_map_str.items()} + + def origin2compact(self, idx): + return self.origin2compact_mapper[int(idx)] + + def compact2origin(self, idx): + return self.compact2origin_mapper[int(idx)] + + +def to_device(item, device): + if isinstance(item, torch.Tensor): + return item.to(device) + elif isinstance(item, list): + return [to_device(i, device) for i in item] + elif isinstance(item, dict): + return {k: to_device(v, device) for k, v in item.items()} + else: + raise NotImplementedError( + "Call Shilong if you use other containers! type: {}".format(type(item)) + ) + + +# +def get_gaussian_mean(x, axis, other_axis, softmax=True): + """ + + Args: + x (float): Input images(BxCxHxW) + axis (int): The index for weighted mean + other_axis (int): The other index + + Returns: weighted index for axis, BxC + + """ + mat2line = torch.sum(x, axis=other_axis) + # mat2line = mat2line / mat2line.mean() * 10 + if softmax: + u = torch.softmax(mat2line, axis=2) + else: + u = mat2line / (mat2line.sum(2, keepdim=True) + 1e-6) + size = x.shape[axis] + ind = torch.linspace(0, 1, size).to(x.device) + batch = x.shape[0] + channel = x.shape[1] + index = ind.repeat([batch, channel, 1]) + mean_position = torch.sum(index * u, dim=2) + return mean_position + + +def get_expected_points_from_map(hm, softmax=True): + """get_gaussian_map_from_points + B,C,H,W -> B,N,2 float(0, 1) float(0, 1) + softargmax function + + Args: + hm (float): Input images(BxCxHxW) + + Returns: + weighted index for axis, BxCx2. float between 0 and 1. + + """ + # hm = 10*hm + B, C, H, W = hm.shape + y_mean = get_gaussian_mean(hm, 2, 3, softmax=softmax) # B,C + x_mean = get_gaussian_mean(hm, 3, 2, softmax=softmax) # B,C + # return torch.cat((x_mean.unsqueeze(-1), y_mean.unsqueeze(-1)), 2) + return torch.stack([x_mean, y_mean], dim=2) + + +# Positional encoding (section 5.1) +# borrow from nerf +class Embedder: + def __init__(self, **kwargs): + self.kwargs = kwargs + self.create_embedding_fn() + + def create_embedding_fn(self): + embed_fns = [] + d = self.kwargs["input_dims"] + out_dim = 0 + if self.kwargs["include_input"]: + embed_fns.append(lambda x: x) + out_dim += d + + max_freq = self.kwargs["max_freq_log2"] + N_freqs = self.kwargs["num_freqs"] + + if self.kwargs["log_sampling"]: + freq_bands = 2.0 ** torch.linspace(0.0, max_freq, steps=N_freqs) + else: + freq_bands = torch.linspace(2.0**0.0, 2.0**max_freq, steps=N_freqs) + + for freq in freq_bands: + for p_fn in self.kwargs["periodic_fns"]: + embed_fns.append(lambda x, p_fn=p_fn, freq=freq: p_fn(x * freq)) + out_dim += d + + self.embed_fns = embed_fns + self.out_dim = out_dim + + def embed(self, inputs): + return torch.cat([fn(inputs) for fn in self.embed_fns], -1) + + +def get_embedder(multires, i=0): + import torch.nn as nn + + if i == -1: + return nn.Identity(), 3 + + embed_kwargs = { + "include_input": True, + "input_dims": 3, + "max_freq_log2": multires - 1, + "num_freqs": multires, + "log_sampling": True, + "periodic_fns": [torch.sin, torch.cos], + } + + embedder_obj = Embedder(**embed_kwargs) + embed = lambda x, eo=embedder_obj: eo.embed(x) + return embed, embedder_obj.out_dim + + +class APOPMeter: + def __init__(self) -> None: + self.tp = 0 + self.fp = 0 + self.tn = 0 + self.fn = 0 + + def update(self, pred, gt): + """ + Input: + pred, gt: Tensor() + """ + assert pred.shape == gt.shape + self.tp += torch.logical_and(pred == 1, gt == 1).sum().item() + self.fp += torch.logical_and(pred == 1, gt == 0).sum().item() + self.tn += torch.logical_and(pred == 0, gt == 0).sum().item() + self.tn += torch.logical_and(pred == 1, gt == 0).sum().item() + + def update_cm(self, tp, fp, tn, fn): + self.tp += tp + self.fp += fp + self.tn += tn + self.tn += fn + + +def inverse_sigmoid(x, eps=1e-5): + x = x.clamp(min=0, max=1) + x1 = x.clamp(min=eps) + x2 = (1 - x).clamp(min=eps) + return torch.log(x1 / x2) + + +def get_raw_dict(args): + """ + return the dicf contained in args. + + e.g: + >>> with open(path, 'w') as f: + json.dump(get_raw_dict(args), f, indent=2) + """ + if isinstance(args, argparse.Namespace): + return vars(args) + elif isinstance(args, dict): + return args + elif isinstance(args, SLConfig): + return args._cfg_dict + else: + raise NotImplementedError("Unknown type {}".format(type(args))) + + +def stat_tensors(tensor): + assert tensor.dim() == 1 + tensor_sm = tensor.softmax(0) + entropy = (tensor_sm * torch.log(tensor_sm + 1e-9)).sum() + + return { + "max": tensor.max(), + "min": tensor.min(), + "mean": tensor.mean(), + "var": tensor.var(), + "std": tensor.var() ** 0.5, + "entropy": entropy, + } + + +class NiceRepr: + """Inherit from this class and define ``__nice__`` to "nicely" print your + objects. + + Defines ``__str__`` and ``__repr__`` in terms of ``__nice__`` function + Classes that inherit from :class:`NiceRepr` should redefine ``__nice__``. + If the inheriting class has a ``__len__``, method then the default + ``__nice__`` method will return its length. + + Example: + >>> class Foo(NiceRepr): + ... def __nice__(self): + ... return 'info' + >>> foo = Foo() + >>> assert str(foo) == '' + >>> assert repr(foo).startswith('>> class Bar(NiceRepr): + ... pass + >>> bar = Bar() + >>> import pytest + >>> with pytest.warns(None) as record: + >>> assert 'object at' in str(bar) + >>> assert 'object at' in repr(bar) + + Example: + >>> class Baz(NiceRepr): + ... def __len__(self): + ... return 5 + >>> baz = Baz() + >>> assert str(baz) == '' + """ + + def __nice__(self): + """str: a "nice" summary string describing this module""" + if hasattr(self, "__len__"): + # It is a common pattern for objects to use __len__ in __nice__ + # As a convenience we define a default __nice__ for these objects + return str(len(self)) + else: + # In all other cases force the subclass to overload __nice__ + raise NotImplementedError(f"Define the __nice__ method for {self.__class__!r}") + + def __repr__(self): + """str: the string of the module""" + try: + nice = self.__nice__() + classname = self.__class__.__name__ + return f"<{classname}({nice}) at {hex(id(self))}>" + except NotImplementedError as ex: + warnings.warn(str(ex), category=RuntimeWarning) + return object.__repr__(self) + + def __str__(self): + """str: the string of the module""" + try: + classname = self.__class__.__name__ + nice = self.__nice__() + return f"<{classname}({nice})>" + except NotImplementedError as ex: + warnings.warn(str(ex), category=RuntimeWarning) + return object.__repr__(self) + + +def ensure_rng(rng=None): + """Coerces input into a random number generator. + + If the input is None, then a global random state is returned. + + If the input is a numeric value, then that is used as a seed to construct a + random state. Otherwise the input is returned as-is. + + Adapted from [1]_. + + Args: + rng (int | numpy.random.RandomState | None): + if None, then defaults to the global rng. Otherwise this can be an + integer or a RandomState class + Returns: + (numpy.random.RandomState) : rng - + a numpy random number generator + + References: + .. [1] https://gitlab.kitware.com/computer-vision/kwarray/blob/master/kwarray/util_random.py#L270 # noqa: E501 + """ + + if rng is None: + rng = np.random.mtrand._rand + elif isinstance(rng, int): + rng = np.random.RandomState(rng) + else: + rng = rng + return rng + + +def random_boxes(num=1, scale=1, rng=None): + """Simple version of ``kwimage.Boxes.random`` + + Returns: + Tensor: shape (n, 4) in x1, y1, x2, y2 format. + + References: + https://gitlab.kitware.com/computer-vision/kwimage/blob/master/kwimage/structs/boxes.py#L1390 + + Example: + >>> num = 3 + >>> scale = 512 + >>> rng = 0 + >>> boxes = random_boxes(num, scale, rng) + >>> print(boxes) + tensor([[280.9925, 278.9802, 308.6148, 366.1769], + [216.9113, 330.6978, 224.0446, 456.5878], + [405.3632, 196.3221, 493.3953, 270.7942]]) + """ + rng = ensure_rng(rng) + + tlbr = rng.rand(num, 4).astype(np.float32) + + tl_x = np.minimum(tlbr[:, 0], tlbr[:, 2]) + tl_y = np.minimum(tlbr[:, 1], tlbr[:, 3]) + br_x = np.maximum(tlbr[:, 0], tlbr[:, 2]) + br_y = np.maximum(tlbr[:, 1], tlbr[:, 3]) + + tlbr[:, 0] = tl_x * scale + tlbr[:, 1] = tl_y * scale + tlbr[:, 2] = br_x * scale + tlbr[:, 3] = br_y * scale + + boxes = torch.from_numpy(tlbr) + return boxes + + +class ModelEma(torch.nn.Module): + def __init__(self, model, decay=0.9997, device=None): + super(ModelEma, self).__init__() + # make a copy of the model for accumulating moving average of weights + self.module = deepcopy(model) + self.module.eval() + + # import ipdb; ipdb.set_trace() + + self.decay = decay + self.device = device # perform ema on different device from model if set + if self.device is not None: + self.module.to(device=device) + + def _update(self, model, update_fn): + with torch.no_grad(): + for ema_v, model_v in zip( + self.module.state_dict().values(), model.state_dict().values() + ): + if self.device is not None: + model_v = model_v.to(device=self.device) + ema_v.copy_(update_fn(ema_v, model_v)) + + def update(self, model): + self._update(model, update_fn=lambda e, m: self.decay * e + (1.0 - self.decay) * m) + + def set(self, model): + self._update(model, update_fn=lambda e, m: m) + + +class BestMetricSingle: + def __init__(self, init_res=0.0, better="large") -> None: + self.init_res = init_res + self.best_res = init_res + self.best_ep = -1 + + self.better = better + assert better in ["large", "small"] + + def isbetter(self, new_res, old_res): + if self.better == "large": + return new_res > old_res + if self.better == "small": + return new_res < old_res + + def update(self, new_res, ep): + if self.isbetter(new_res, self.best_res): + self.best_res = new_res + self.best_ep = ep + return True + return False + + def __str__(self) -> str: + return "best_res: {}\t best_ep: {}".format(self.best_res, self.best_ep) + + def __repr__(self) -> str: + return self.__str__() + + def summary(self) -> dict: + return { + "best_res": self.best_res, + "best_ep": self.best_ep, + } + + +class BestMetricHolder: + def __init__(self, init_res=0.0, better="large", use_ema=False) -> None: + self.best_all = BestMetricSingle(init_res, better) + self.use_ema = use_ema + if use_ema: + self.best_ema = BestMetricSingle(init_res, better) + self.best_regular = BestMetricSingle(init_res, better) + + def update(self, new_res, epoch, is_ema=False): + """ + return if the results is the best. + """ + if not self.use_ema: + return self.best_all.update(new_res, epoch) + else: + if is_ema: + self.best_ema.update(new_res, epoch) + return self.best_all.update(new_res, epoch) + else: + self.best_regular.update(new_res, epoch) + return self.best_all.update(new_res, epoch) + + def summary(self): + if not self.use_ema: + return self.best_all.summary() + + res = {} + res.update({f"all_{k}": v for k, v in self.best_all.summary().items()}) + res.update({f"regular_{k}": v for k, v in self.best_regular.summary().items()}) + res.update({f"ema_{k}": v for k, v in self.best_ema.summary().items()}) + return res + + def __repr__(self) -> str: + return json.dumps(self.summary(), indent=2) + + def __str__(self) -> str: + return self.__repr__() + + +def targets_to(targets: List[Dict[str, Any]], device): + """Moves the target dicts to the given device.""" + excluded_keys = [ + "questionId", + "tokens_positive", + "strings_positive", + "tokens", + "dataset_name", + "sentence_id", + "original_img_id", + "nb_eval", + "task_id", + "original_id", + "token_span", + "caption", + "dataset_type", + ] + return [ + {k: v.to(device) if k not in excluded_keys else v for k, v in t.items()} for t in targets + ] + + +def get_phrases_from_posmap( + posmap: torch.BoolTensor, tokenized: Dict, tokenizer: AutoTokenizer, left_idx: int = 0, right_idx: int = 255 +): + assert isinstance(posmap, torch.Tensor), "posmap must be torch.Tensor" + if posmap.dim() == 1: + posmap[0: left_idx + 1] = False + posmap[right_idx:] = False + non_zero_idx = posmap.nonzero(as_tuple=True)[0].tolist() + token_ids = [tokenized["input_ids"][i] for i in non_zero_idx] + return tokenizer.decode(token_ids) + else: + raise NotImplementedError("posmap must be 1-dim") diff --git a/groundingdino/util/visualizer.py b/groundingdino/util/visualizer.py new file mode 100644 index 0000000000000000000000000000000000000000..7a1b7b101e9b73f75f9136bc67f2063c7c1cf1c1 --- /dev/null +++ b/groundingdino/util/visualizer.py @@ -0,0 +1,318 @@ +# -*- coding: utf-8 -*- +""" +@File : visualizer.py +@Time : 2022/04/05 11:39:33 +@Author : Shilong Liu +@Contact : slongliu86@gmail.com +""" + +import datetime +import os + +import cv2 +import matplotlib.pyplot as plt +import numpy as np +import torch +from matplotlib import transforms +from matplotlib.collections import PatchCollection +from matplotlib.patches import Polygon +from pycocotools import mask as maskUtils + + +def renorm( + img: torch.FloatTensor, mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225] +) -> torch.FloatTensor: + # img: tensor(3,H,W) or tensor(B,3,H,W) + # return: same as img + assert img.dim() == 3 or img.dim() == 4, "img.dim() should be 3 or 4 but %d" % img.dim() + if img.dim() == 3: + assert img.size(0) == 3, 'img.size(0) shoule be 3 but "%d". (%s)' % ( + img.size(0), + str(img.size()), + ) + img_perm = img.permute(1, 2, 0) + mean = torch.Tensor(mean) + std = torch.Tensor(std) + img_res = img_perm * std + mean + return img_res.permute(2, 0, 1) + else: # img.dim() == 4 + assert img.size(1) == 3, 'img.size(1) shoule be 3 but "%d". (%s)' % ( + img.size(1), + str(img.size()), + ) + img_perm = img.permute(0, 2, 3, 1) + mean = torch.Tensor(mean) + std = torch.Tensor(std) + img_res = img_perm * std + mean + return img_res.permute(0, 3, 1, 2) + + +class ColorMap: + def __init__(self, basergb=[255, 255, 0]): + self.basergb = np.array(basergb) + + def __call__(self, attnmap): + # attnmap: h, w. np.uint8. + # return: h, w, 4. np.uint8. + assert attnmap.dtype == np.uint8 + h, w = attnmap.shape + res = self.basergb.copy() + res = res[None][None].repeat(h, 0).repeat(w, 1) # h, w, 3 + attn1 = attnmap.copy()[..., None] # h, w, 1 + res = np.concatenate((res, attn1), axis=-1).astype(np.uint8) + return res + + +def rainbow_text(x, y, ls, lc, **kw): + """ + Take a list of strings ``ls`` and colors ``lc`` and place them next to each + other, with text ls[i] being shown in color lc[i]. + + This example shows how to do both vertical and horizontal text, and will + pass all keyword arguments to plt.text, so you can set the font size, + family, etc. + """ + t = plt.gca().transData + fig = plt.gcf() + plt.show() + + # horizontal version + for s, c in zip(ls, lc): + text = plt.text(x, y, " " + s + " ", color=c, transform=t, **kw) + text.draw(fig.canvas.get_renderer()) + ex = text.get_window_extent() + t = transforms.offset_copy(text._transform, x=ex.width, units="dots") + + # #vertical version + # for s,c in zip(ls,lc): + # text = plt.text(x,y," "+s+" ",color=c, transform=t, + # rotation=90,va='bottom',ha='center',**kw) + # text.draw(fig.canvas.get_renderer()) + # ex = text.get_window_extent() + # t = transforms.offset_copy(text._transform, y=ex.height, units='dots') + + +class COCOVisualizer: + def __init__(self, coco=None, tokenlizer=None) -> None: + self.coco = coco + + def visualize(self, img, tgt, caption=None, dpi=180, savedir="vis"): + """ + img: tensor(3, H, W) + tgt: make sure they are all on cpu. + must have items: 'image_id', 'boxes', 'size' + """ + plt.figure(dpi=dpi) + plt.rcParams["font.size"] = "5" + ax = plt.gca() + img = renorm(img).permute(1, 2, 0) + # if os.environ.get('IPDB_SHILONG_DEBUG', None) == 'INFO': + # import ipdb; ipdb.set_trace() + ax.imshow(img) + + self.addtgt(tgt) + + if tgt is None: + image_id = 0 + elif "image_id" not in tgt: + image_id = 0 + else: + image_id = tgt["image_id"] + + if caption is None: + savename = "{}/{}-{}.png".format( + savedir, int(image_id), str(datetime.datetime.now()).replace(" ", "-") + ) + else: + savename = "{}/{}-{}-{}.png".format( + savedir, caption, int(image_id), str(datetime.datetime.now()).replace(" ", "-") + ) + print("savename: {}".format(savename)) + os.makedirs(os.path.dirname(savename), exist_ok=True) + plt.savefig(savename) + plt.close() + + def addtgt(self, tgt): + """ """ + if tgt is None or not "boxes" in tgt: + ax = plt.gca() + + if "caption" in tgt: + ax.set_title(tgt["caption"], wrap=True) + + ax.set_axis_off() + return + + ax = plt.gca() + H, W = tgt["size"] + numbox = tgt["boxes"].shape[0] + + color = [] + polygons = [] + boxes = [] + for box in tgt["boxes"].cpu(): + unnormbbox = box * torch.Tensor([W, H, W, H]) + unnormbbox[:2] -= unnormbbox[2:] / 2 + [bbox_x, bbox_y, bbox_w, bbox_h] = unnormbbox.tolist() + boxes.append([bbox_x, bbox_y, bbox_w, bbox_h]) + poly = [ + [bbox_x, bbox_y], + [bbox_x, bbox_y + bbox_h], + [bbox_x + bbox_w, bbox_y + bbox_h], + [bbox_x + bbox_w, bbox_y], + ] + np_poly = np.array(poly).reshape((4, 2)) + polygons.append(Polygon(np_poly)) + c = (np.random.random((1, 3)) * 0.6 + 0.4).tolist()[0] + color.append(c) + + p = PatchCollection(polygons, facecolor=color, linewidths=0, alpha=0.1) + ax.add_collection(p) + p = PatchCollection(polygons, facecolor="none", edgecolors=color, linewidths=2) + ax.add_collection(p) + + if "strings_positive" in tgt and len(tgt["strings_positive"]) > 0: + assert ( + len(tgt["strings_positive"]) == numbox + ), f"{len(tgt['strings_positive'])} = {numbox}, " + for idx, strlist in enumerate(tgt["strings_positive"]): + cate_id = int(tgt["labels"][idx]) + _string = str(cate_id) + ":" + " ".join(strlist) + bbox_x, bbox_y, bbox_w, bbox_h = boxes[idx] + # ax.text(bbox_x, bbox_y, _string, color='black', bbox={'facecolor': 'yellow', 'alpha': 1.0, 'pad': 1}) + ax.text( + bbox_x, + bbox_y, + _string, + color="black", + bbox={"facecolor": color[idx], "alpha": 0.6, "pad": 1}, + ) + + if "box_label" in tgt: + assert len(tgt["box_label"]) == numbox, f"{len(tgt['box_label'])} = {numbox}, " + for idx, bl in enumerate(tgt["box_label"]): + _string = str(bl) + bbox_x, bbox_y, bbox_w, bbox_h = boxes[idx] + # ax.text(bbox_x, bbox_y, _string, color='black', bbox={'facecolor': 'yellow', 'alpha': 1.0, 'pad': 1}) + ax.text( + bbox_x, + bbox_y, + _string, + color="black", + bbox={"facecolor": color[idx], "alpha": 0.6, "pad": 1}, + ) + + if "caption" in tgt: + ax.set_title(tgt["caption"], wrap=True) + # plt.figure() + # rainbow_text(0.0,0.0,"all unicorns poop rainbows ! ! !".split(), + # ['red', 'orange', 'brown', 'green', 'blue', 'purple', 'black']) + + if "attn" in tgt: + # if os.environ.get('IPDB_SHILONG_DEBUG', None) == 'INFO': + # import ipdb; ipdb.set_trace() + if isinstance(tgt["attn"], tuple): + tgt["attn"] = [tgt["attn"]] + for item in tgt["attn"]: + attn_map, basergb = item + attn_map = (attn_map - attn_map.min()) / (attn_map.max() - attn_map.min() + 1e-3) + attn_map = (attn_map * 255).astype(np.uint8) + cm = ColorMap(basergb) + heatmap = cm(attn_map) + ax.imshow(heatmap) + ax.set_axis_off() + + def showAnns(self, anns, draw_bbox=False): + """ + Display the specified annotations. + :param anns (array of object): annotations to display + :return: None + """ + if len(anns) == 0: + return 0 + if "segmentation" in anns[0] or "keypoints" in anns[0]: + datasetType = "instances" + elif "caption" in anns[0]: + datasetType = "captions" + else: + raise Exception("datasetType not supported") + if datasetType == "instances": + ax = plt.gca() + ax.set_autoscale_on(False) + polygons = [] + color = [] + for ann in anns: + c = (np.random.random((1, 3)) * 0.6 + 0.4).tolist()[0] + if "segmentation" in ann: + if type(ann["segmentation"]) == list: + # polygon + for seg in ann["segmentation"]: + poly = np.array(seg).reshape((int(len(seg) / 2), 2)) + polygons.append(Polygon(poly)) + color.append(c) + else: + # mask + t = self.imgs[ann["image_id"]] + if type(ann["segmentation"]["counts"]) == list: + rle = maskUtils.frPyObjects( + [ann["segmentation"]], t["height"], t["width"] + ) + else: + rle = [ann["segmentation"]] + m = maskUtils.decode(rle) + img = np.ones((m.shape[0], m.shape[1], 3)) + if ann["iscrowd"] == 1: + color_mask = np.array([2.0, 166.0, 101.0]) / 255 + if ann["iscrowd"] == 0: + color_mask = np.random.random((1, 3)).tolist()[0] + for i in range(3): + img[:, :, i] = color_mask[i] + ax.imshow(np.dstack((img, m * 0.5))) + if "keypoints" in ann and type(ann["keypoints"]) == list: + # turn skeleton into zero-based index + sks = np.array(self.loadCats(ann["category_id"])[0]["skeleton"]) - 1 + kp = np.array(ann["keypoints"]) + x = kp[0::3] + y = kp[1::3] + v = kp[2::3] + for sk in sks: + if np.all(v[sk] > 0): + plt.plot(x[sk], y[sk], linewidth=3, color=c) + plt.plot( + x[v > 0], + y[v > 0], + "o", + markersize=8, + markerfacecolor=c, + markeredgecolor="k", + markeredgewidth=2, + ) + plt.plot( + x[v > 1], + y[v > 1], + "o", + markersize=8, + markerfacecolor=c, + markeredgecolor=c, + markeredgewidth=2, + ) + + if draw_bbox: + [bbox_x, bbox_y, bbox_w, bbox_h] = ann["bbox"] + poly = [ + [bbox_x, bbox_y], + [bbox_x, bbox_y + bbox_h], + [bbox_x + bbox_w, bbox_y + bbox_h], + [bbox_x + bbox_w, bbox_y], + ] + np_poly = np.array(poly).reshape((4, 2)) + polygons.append(Polygon(np_poly)) + color.append(c) + + # p = PatchCollection(polygons, facecolor=color, linewidths=0, alpha=0.4) + # ax.add_collection(p) + p = PatchCollection(polygons, facecolor="none", edgecolors=color, linewidths=2) + ax.add_collection(p) + elif datasetType == "captions": + for ann in anns: + print(ann["caption"]) diff --git a/groundingdino/util/vl_utils.py b/groundingdino/util/vl_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..c91bb02f584398f08a28e6b7719e2b99f6e28616 --- /dev/null +++ b/groundingdino/util/vl_utils.py @@ -0,0 +1,100 @@ +import os +import random +from typing import List + +import torch + + +def create_positive_map_from_span(tokenized, token_span, max_text_len=256): + """construct a map such that positive_map[i,j] = True iff box i is associated to token j + Input: + - tokenized: + - input_ids: Tensor[1, ntokens] + - attention_mask: Tensor[1, ntokens] + - token_span: list with length num_boxes. + - each item: [start_idx, end_idx] + """ + positive_map = torch.zeros((len(token_span), max_text_len), dtype=torch.float) + for j, tok_list in enumerate(token_span): + for (beg, end) in tok_list: + beg_pos = tokenized.char_to_token(beg) + end_pos = tokenized.char_to_token(end - 1) + if beg_pos is None: + try: + beg_pos = tokenized.char_to_token(beg + 1) + if beg_pos is None: + beg_pos = tokenized.char_to_token(beg + 2) + except: + beg_pos = None + if end_pos is None: + try: + end_pos = tokenized.char_to_token(end - 2) + if end_pos is None: + end_pos = tokenized.char_to_token(end - 3) + except: + end_pos = None + if beg_pos is None or end_pos is None: + continue + + assert beg_pos is not None and end_pos is not None + if os.environ.get("SHILONG_DEBUG_ONLY_ONE_POS", None) == "TRUE": + positive_map[j, beg_pos] = 1 + break + else: + positive_map[j, beg_pos : end_pos + 1].fill_(1) + + return positive_map / (positive_map.sum(-1)[:, None] + 1e-6) + + +def build_captions_and_token_span(cat_list, force_lowercase): + """ + Return: + captions: str + cat2tokenspan: dict + { + 'dog': [[0, 2]], + ... + } + """ + + cat2tokenspan = {} + captions = "" + for catname in cat_list: + class_name = catname + if force_lowercase: + class_name = class_name.lower() + if "/" in class_name: + class_name_list: List = class_name.strip().split("/") + class_name_list.append(class_name) + class_name: str = random.choice(class_name_list) + + tokens_positive_i = [] + subnamelist = [i.strip() for i in class_name.strip().split(" ")] + for subname in subnamelist: + if len(subname) == 0: + continue + if len(captions) > 0: + captions = captions + " " + strat_idx = len(captions) + end_idx = strat_idx + len(subname) + tokens_positive_i.append([strat_idx, end_idx]) + captions = captions + subname + + if len(tokens_positive_i) > 0: + captions = captions + " ." + cat2tokenspan[class_name] = tokens_positive_i + + return captions, cat2tokenspan + + +def build_id2posspan_and_caption(category_dict: dict): + """Build id2pos_span and caption from category_dict + + Args: + category_dict (dict): category_dict + """ + cat_list = [item["name"].lower() for item in category_dict] + id2catname = {item["id"]: item["name"].lower() for item in category_dict} + caption, cat2posspan = build_captions_and_token_span(cat_list, force_lowercase=True) + id2posspan = {catid: cat2posspan[catname] for catid, catname in id2catname.items()} + return id2posspan, caption diff --git a/lego.jpg b/lego.jpg new file mode 100644 index 0000000000000000000000000000000000000000..8426603c979e7c54afff3399477d566a9642c58f Binary files /dev/null and b/lego.jpg differ diff --git a/log.csv b/log.csv new file mode 100644 index 0000000000000000000000000000000000000000..c67c2c5d6cde2cef527c5ec88fe9a4a59bc58764 --- /dev/null +++ b/log.csv @@ -0,0 +1,3 @@ +flagged/Input Image/d206f8d836baf220ac41/HVITa2016a_000011.JPG,bird,,flagged/Detected Instances/9949e21c2f5bacf777a1/image.webp,30,,,2024-06-24 16:27:43.463142 +flagged/Input Image/86a081dc917ef9d837c7/HVITa2016a_000011.JPG,,"{""image"": ""flagged/Specify object to count with visual exemplars here/64abdac51a65819724a6/HVITa2016a_000011.JPG"", ""points"": [[1230.0, 118.0, 2.0, 1292.0, 201.0, 3.0], [830.0, 550.0, 2.0, 906.0, 623.0, 3.0]]}",flagged/Detected Instances/d92c3d3991cfcefde001/image.webp,33,,,2024-06-24 16:28:22.627606 +flagged/Input Image/35151af517a911bcc223/HVITa2016a_000011.JPG,bird,"{""image"": ""flagged/Specify object to count with visual exemplars here/4f611be7fe99047414ae/HVITa2016a_000011.JPG"", ""points"": [[1230.0, 118.0, 2.0, 1292.0, 201.0, 3.0], [830.0, 550.0, 2.0, 906.0, 623.0, 3.0]]}",flagged/Detected Instances/10125734c7afa38ed087/image.webp,31,,,2024-06-24 16:28:36.193649 diff --git a/models/GroundingDINO/__init__.py b/models/GroundingDINO/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..2af819d61d589cfec2e0ca46612a7456f42b831a --- /dev/null +++ b/models/GroundingDINO/__init__.py @@ -0,0 +1,15 @@ +# ------------------------------------------------------------------------ +# Grounding DINO +# url: https://github.com/IDEA-Research/GroundingDINO +# Copyright (c) 2023 IDEA. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------ +# Conditional DETR +# Copyright (c) 2021 Microsoft. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------ +# Copied from DETR (https://github.com/facebookresearch/detr) +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +# ------------------------------------------------------------------------ + +from .groundingdino import build_groundingdino diff --git a/models/GroundingDINO/__pycache__/__init__.cpython-38.pyc b/models/GroundingDINO/__pycache__/__init__.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3541129c2348d418958266c61e3790ab60346e6f Binary files /dev/null and b/models/GroundingDINO/__pycache__/__init__.cpython-38.pyc differ diff --git a/models/GroundingDINO/__pycache__/__init__.cpython-39.pyc b/models/GroundingDINO/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..46d1f96aae19009ce53ba13b4b41dcdf21fe2638 Binary files /dev/null and b/models/GroundingDINO/__pycache__/__init__.cpython-39.pyc differ diff --git a/models/GroundingDINO/__pycache__/bertwarper.cpython-38.pyc b/models/GroundingDINO/__pycache__/bertwarper.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1f19d4ad2b78f414cbd64e885cf7f6503bc00ed3 Binary files /dev/null and b/models/GroundingDINO/__pycache__/bertwarper.cpython-38.pyc differ diff --git a/models/GroundingDINO/__pycache__/bertwarper.cpython-39.pyc b/models/GroundingDINO/__pycache__/bertwarper.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..484b97a85acb08e51e057e34137467ba6a2e6379 Binary files /dev/null and b/models/GroundingDINO/__pycache__/bertwarper.cpython-39.pyc differ diff --git a/models/GroundingDINO/__pycache__/fuse_modules.cpython-38.pyc b/models/GroundingDINO/__pycache__/fuse_modules.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8d8c7e88e3e0a06d040be916d5162ec262bdc72d Binary files /dev/null and b/models/GroundingDINO/__pycache__/fuse_modules.cpython-38.pyc differ diff --git a/models/GroundingDINO/__pycache__/fuse_modules.cpython-39.pyc b/models/GroundingDINO/__pycache__/fuse_modules.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d48dfa8a3c7fad2ce0e8c6da32e4d84c028ad55a Binary files /dev/null and b/models/GroundingDINO/__pycache__/fuse_modules.cpython-39.pyc differ diff --git a/models/GroundingDINO/__pycache__/groundingdino.cpython-38.pyc b/models/GroundingDINO/__pycache__/groundingdino.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..30c46f19fe3ff77f89bf5a0f64562ac1739f3a9d Binary files /dev/null and b/models/GroundingDINO/__pycache__/groundingdino.cpython-38.pyc differ diff --git a/models/GroundingDINO/__pycache__/groundingdino.cpython-39.pyc b/models/GroundingDINO/__pycache__/groundingdino.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..74aa1596b83ccf5ddd3bf6937b155a0c6d2c5754 Binary files /dev/null and b/models/GroundingDINO/__pycache__/groundingdino.cpython-39.pyc differ diff --git a/models/GroundingDINO/__pycache__/matcher.cpython-38.pyc b/models/GroundingDINO/__pycache__/matcher.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e92d533d6fc9e151538073d3fddbbb363f37da85 Binary files /dev/null and b/models/GroundingDINO/__pycache__/matcher.cpython-38.pyc differ diff --git a/models/GroundingDINO/__pycache__/matcher.cpython-39.pyc b/models/GroundingDINO/__pycache__/matcher.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d07d9d6df6fa469d61c58132e2b5a34cb3410b39 Binary files /dev/null and b/models/GroundingDINO/__pycache__/matcher.cpython-39.pyc differ diff --git a/models/GroundingDINO/__pycache__/mlp_loca.cpython-39.pyc b/models/GroundingDINO/__pycache__/mlp_loca.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a2459cf30e9b8ab959961266b5cbdc57bb84271c Binary files /dev/null and b/models/GroundingDINO/__pycache__/mlp_loca.cpython-39.pyc differ diff --git a/models/GroundingDINO/__pycache__/ms_deform_attn.cpython-38.pyc b/models/GroundingDINO/__pycache__/ms_deform_attn.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..20ae50851294c327a9a8797809f6bd8f383c69bf Binary files /dev/null and b/models/GroundingDINO/__pycache__/ms_deform_attn.cpython-38.pyc differ diff --git a/models/GroundingDINO/__pycache__/ms_deform_attn.cpython-39.pyc b/models/GroundingDINO/__pycache__/ms_deform_attn.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..815f1f95edc5acf6d2d631274719b9243029bdb1 Binary files /dev/null and b/models/GroundingDINO/__pycache__/ms_deform_attn.cpython-39.pyc differ diff --git a/models/GroundingDINO/__pycache__/positional_encoding_loca.cpython-39.pyc b/models/GroundingDINO/__pycache__/positional_encoding_loca.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a1f14bc4858e0281cf2fccbeb12c736619186121 Binary files /dev/null and b/models/GroundingDINO/__pycache__/positional_encoding_loca.cpython-39.pyc differ diff --git a/models/GroundingDINO/__pycache__/transformer.cpython-38.pyc b/models/GroundingDINO/__pycache__/transformer.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8f0da641c6af09e6db675787ba589ecb2658d424 Binary files /dev/null and b/models/GroundingDINO/__pycache__/transformer.cpython-38.pyc differ diff --git a/models/GroundingDINO/__pycache__/transformer.cpython-39.pyc b/models/GroundingDINO/__pycache__/transformer.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..30b07dbffb832317d47e1252fb9f6a85e690821f Binary files /dev/null and b/models/GroundingDINO/__pycache__/transformer.cpython-39.pyc differ diff --git a/models/GroundingDINO/__pycache__/transformer_loca.cpython-39.pyc b/models/GroundingDINO/__pycache__/transformer_loca.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1b40efa4136a875aae9ca8f412c02e9e73e1e331 Binary files /dev/null and b/models/GroundingDINO/__pycache__/transformer_loca.cpython-39.pyc differ diff --git a/models/GroundingDINO/__pycache__/transformer_vanilla.cpython-38.pyc b/models/GroundingDINO/__pycache__/transformer_vanilla.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f5cac6ac7ef19c185fa80e30ff40ec60a697a72e Binary files /dev/null and b/models/GroundingDINO/__pycache__/transformer_vanilla.cpython-38.pyc differ diff --git a/models/GroundingDINO/__pycache__/transformer_vanilla.cpython-39.pyc b/models/GroundingDINO/__pycache__/transformer_vanilla.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..161972d091c28409114fbe01fdef4d62328db55b Binary files /dev/null and b/models/GroundingDINO/__pycache__/transformer_vanilla.cpython-39.pyc differ diff --git a/models/GroundingDINO/__pycache__/utils.cpython-38.pyc b/models/GroundingDINO/__pycache__/utils.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b4ba05e10b27556530df049ac76d1c515df41f6d Binary files /dev/null and b/models/GroundingDINO/__pycache__/utils.cpython-38.pyc differ diff --git a/models/GroundingDINO/__pycache__/utils.cpython-39.pyc b/models/GroundingDINO/__pycache__/utils.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..62ce3c777ca9baddd91c90e907d8b22c10ad6fad Binary files /dev/null and b/models/GroundingDINO/__pycache__/utils.cpython-39.pyc differ diff --git a/models/GroundingDINO/backbone/__init__.py b/models/GroundingDINO/backbone/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..76e4b272b479a26c63d120c818c140870cd8c287 --- /dev/null +++ b/models/GroundingDINO/backbone/__init__.py @@ -0,0 +1 @@ +from .backbone import build_backbone diff --git a/models/GroundingDINO/backbone/__pycache__/__init__.cpython-38.pyc b/models/GroundingDINO/backbone/__pycache__/__init__.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7bcca021cc5a6fc65cb4224dd96941755fff3835 Binary files /dev/null and b/models/GroundingDINO/backbone/__pycache__/__init__.cpython-38.pyc differ diff --git a/models/GroundingDINO/backbone/__pycache__/__init__.cpython-39.pyc b/models/GroundingDINO/backbone/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5d9a8d98e8aaf2e70be64928deb00b0f0495e029 Binary files /dev/null and b/models/GroundingDINO/backbone/__pycache__/__init__.cpython-39.pyc differ diff --git a/models/GroundingDINO/backbone/__pycache__/backbone.cpython-38.pyc b/models/GroundingDINO/backbone/__pycache__/backbone.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..28854a4ed50574a475ae1deba8b4b4324313976b Binary files /dev/null and b/models/GroundingDINO/backbone/__pycache__/backbone.cpython-38.pyc differ diff --git a/models/GroundingDINO/backbone/__pycache__/backbone.cpython-39.pyc b/models/GroundingDINO/backbone/__pycache__/backbone.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0caca112179e7ded327cd45d0d6a60f5bb900791 Binary files /dev/null and b/models/GroundingDINO/backbone/__pycache__/backbone.cpython-39.pyc differ diff --git a/models/GroundingDINO/backbone/__pycache__/position_encoding.cpython-38.pyc b/models/GroundingDINO/backbone/__pycache__/position_encoding.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8e4152cd12ebd6ab1bfae3c9cac459032bda4374 Binary files /dev/null and b/models/GroundingDINO/backbone/__pycache__/position_encoding.cpython-38.pyc differ diff --git a/models/GroundingDINO/backbone/__pycache__/position_encoding.cpython-39.pyc b/models/GroundingDINO/backbone/__pycache__/position_encoding.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7387f4071ed94560ee87a2a03e422805d14c00ea Binary files /dev/null and b/models/GroundingDINO/backbone/__pycache__/position_encoding.cpython-39.pyc differ diff --git a/models/GroundingDINO/backbone/__pycache__/swin_transformer.cpython-38.pyc b/models/GroundingDINO/backbone/__pycache__/swin_transformer.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..def9e3e4215e1fb5275c50e96ac4d4dcfa527008 Binary files /dev/null and b/models/GroundingDINO/backbone/__pycache__/swin_transformer.cpython-38.pyc differ diff --git a/models/GroundingDINO/backbone/__pycache__/swin_transformer.cpython-39.pyc b/models/GroundingDINO/backbone/__pycache__/swin_transformer.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6bd071069bb03b015acfa1c19c58cd5434246a52 Binary files /dev/null and b/models/GroundingDINO/backbone/__pycache__/swin_transformer.cpython-39.pyc differ diff --git a/models/GroundingDINO/backbone/backbone.py b/models/GroundingDINO/backbone/backbone.py new file mode 100644 index 0000000000000000000000000000000000000000..c8340c723fad8e07e2fc62daaa3912487498814b --- /dev/null +++ b/models/GroundingDINO/backbone/backbone.py @@ -0,0 +1,221 @@ +# ------------------------------------------------------------------------ +# Grounding DINO +# url: https://github.com/IDEA-Research/GroundingDINO +# Copyright (c) 2023 IDEA. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------ +# Conditional DETR +# Copyright (c) 2021 Microsoft. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------ +# Copied from DETR (https://github.com/facebookresearch/detr) +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +# ------------------------------------------------------------------------ + +""" +Backbone modules. +""" + +from typing import Dict, List + +import torch +import torch.nn.functional as F +import torchvision +from torch import nn +from torchvision.models._utils import IntermediateLayerGetter + +from groundingdino.util.misc import NestedTensor, clean_state_dict, is_main_process + +from .position_encoding import build_position_encoding +from .swin_transformer import build_swin_transformer + + +class FrozenBatchNorm2d(torch.nn.Module): + """ + BatchNorm2d where the batch statistics and the affine parameters are fixed. + + Copy-paste from torchvision.misc.ops with added eps before rqsrt, + without which any other models than torchvision.models.resnet[18,34,50,101] + produce nans. + """ + + def __init__(self, n): + super(FrozenBatchNorm2d, self).__init__() + self.register_buffer("weight", torch.ones(n)) + self.register_buffer("bias", torch.zeros(n)) + self.register_buffer("running_mean", torch.zeros(n)) + self.register_buffer("running_var", torch.ones(n)) + + def _load_from_state_dict( + self, state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys, error_msgs + ): + num_batches_tracked_key = prefix + "num_batches_tracked" + if num_batches_tracked_key in state_dict: + del state_dict[num_batches_tracked_key] + + super(FrozenBatchNorm2d, self)._load_from_state_dict( + state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys, error_msgs + ) + + def forward(self, x): + # move reshapes to the beginning + # to make it fuser-friendly + w = self.weight.reshape(1, -1, 1, 1) + b = self.bias.reshape(1, -1, 1, 1) + rv = self.running_var.reshape(1, -1, 1, 1) + rm = self.running_mean.reshape(1, -1, 1, 1) + eps = 1e-5 + scale = w * (rv + eps).rsqrt() + bias = b - rm * scale + return x * scale + bias + + +class BackboneBase(nn.Module): + def __init__( + self, + backbone: nn.Module, + train_backbone: bool, + num_channels: int, + return_interm_indices: list, + ): + super().__init__() + for name, parameter in backbone.named_parameters(): + if ( + not train_backbone + or "layer2" not in name + and "layer3" not in name + and "layer4" not in name + ): + parameter.requires_grad_(False) + + return_layers = {} + for idx, layer_index in enumerate(return_interm_indices): + return_layers.update( + {"layer{}".format(5 - len(return_interm_indices) + idx): "{}".format(layer_index)} + ) + + # if len: + # if use_stage1_feature: + # return_layers = {"layer1": "0", "layer2": "1", "layer3": "2", "layer4": "3"} + # else: + # return_layers = {"layer2": "0", "layer3": "1", "layer4": "2"} + # else: + # return_layers = {'layer4': "0"} + self.body = IntermediateLayerGetter(backbone, return_layers=return_layers) + self.num_channels = num_channels + + def forward(self, tensor_list: NestedTensor): + xs = self.body(tensor_list.tensors) + out: Dict[str, NestedTensor] = {} + for name, x in xs.items(): + m = tensor_list.mask + assert m is not None + mask = F.interpolate(m[None].float(), size=x.shape[-2:]).to(torch.bool)[0] + out[name] = NestedTensor(x, mask) + # import ipdb; ipdb.set_trace() + return out + + +class Backbone(BackboneBase): + """ResNet backbone with frozen BatchNorm.""" + + def __init__( + self, + name: str, + train_backbone: bool, + dilation: bool, + return_interm_indices: list, + batch_norm=FrozenBatchNorm2d, + ): + if name in ["resnet18", "resnet34", "resnet50", "resnet101"]: + backbone = getattr(torchvision.models, name)( + replace_stride_with_dilation=[False, False, dilation], + pretrained=is_main_process(), + norm_layer=batch_norm, + ) + else: + raise NotImplementedError("Why you can get here with name {}".format(name)) + # num_channels = 512 if name in ('resnet18', 'resnet34') else 2048 + assert name not in ("resnet18", "resnet34"), "Only resnet50 and resnet101 are available." + assert return_interm_indices in [[0, 1, 2, 3], [1, 2, 3], [3]] + num_channels_all = [256, 512, 1024, 2048] + num_channels = num_channels_all[4 - len(return_interm_indices) :] + super().__init__(backbone, train_backbone, num_channels, return_interm_indices) + + +class Joiner(nn.Sequential): + def __init__(self, backbone, position_embedding): + super().__init__(backbone, position_embedding) + + def forward(self, tensor_list: NestedTensor): + xs = self[0](tensor_list) + out: List[NestedTensor] = [] + pos = [] + for name, x in xs.items(): + out.append(x) + # position encoding + pos.append(self[1](x).to(x.tensors.dtype)) + + return out, pos + + +def build_backbone(args): + """ + Useful args: + - backbone: backbone name + - lr_backbone: + - dilation + - return_interm_indices: available: [0,1,2,3], [1,2,3], [3] + - backbone_freeze_keywords: + - use_checkpoint: for swin only for now + + """ + position_embedding = build_position_encoding(args) + train_backbone = True + if not train_backbone: + raise ValueError("Please set lr_backbone > 0") + return_interm_indices = args.return_interm_indices + assert return_interm_indices in [[0, 1, 2, 3], [1, 2, 3], [3]] + args.backbone_freeze_keywords + use_checkpoint = getattr(args, "use_checkpoint", False) + + if args.backbone in ["resnet50", "resnet101"]: + backbone = Backbone( + args.backbone, + train_backbone, + args.dilation, + return_interm_indices, + batch_norm=FrozenBatchNorm2d, + ) + bb_num_channels = backbone.num_channels + elif args.backbone in [ + "swin_T_224_1k", + "swin_B_224_22k", + "swin_B_384_22k", + "swin_L_224_22k", + "swin_L_384_22k", + ]: + pretrain_img_size = int(args.backbone.split("_")[-2]) + backbone = build_swin_transformer( + args.backbone, + pretrain_img_size=pretrain_img_size, + out_indices=tuple(return_interm_indices), + dilation=False, + use_checkpoint=use_checkpoint, + ) + + bb_num_channels = backbone.num_features[4 - len(return_interm_indices) :] + else: + raise NotImplementedError("Unknown backbone {}".format(args.backbone)) + + assert len(bb_num_channels) == len( + return_interm_indices + ), f"len(bb_num_channels) {len(bb_num_channels)} != len(return_interm_indices) {len(return_interm_indices)}" + + model = Joiner(backbone, position_embedding) + model.num_channels = bb_num_channels + assert isinstance( + bb_num_channels, List + ), "bb_num_channels is expected to be a List but {}".format(type(bb_num_channels)) + # import ipdb; ipdb.set_trace() + return model diff --git a/models/GroundingDINO/backbone/position_encoding.py b/models/GroundingDINO/backbone/position_encoding.py new file mode 100644 index 0000000000000000000000000000000000000000..eac7e896bbe85a670824bfe8ef487d0535d5bd99 --- /dev/null +++ b/models/GroundingDINO/backbone/position_encoding.py @@ -0,0 +1,186 @@ +# ------------------------------------------------------------------------ +# Grounding DINO +# url: https://github.com/IDEA-Research/GroundingDINO +# Copyright (c) 2023 IDEA. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------ +# DINO +# Copyright (c) 2022 IDEA. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------ +# Conditional DETR +# Copyright (c) 2021 Microsoft. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------ +# Copied from DETR (https://github.com/facebookresearch/detr) +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +# ------------------------------------------------------------------------ + +""" +Various positional encodings for the transformer. +""" +import math + +import torch +from torch import nn + +from groundingdino.util.misc import NestedTensor + + +class PositionEmbeddingSine(nn.Module): + """ + This is a more standard version of the position embedding, very similar to the one + used by the Attention is all you need paper, generalized to work on images. + """ + + def __init__(self, num_pos_feats=64, temperature=10000, normalize=False, scale=None): + super().__init__() + self.num_pos_feats = num_pos_feats + self.temperature = temperature + self.normalize = normalize + if scale is not None and normalize is False: + raise ValueError("normalize should be True if scale is passed") + if scale is None: + scale = 2 * math.pi + self.scale = scale + + def forward(self, tensor_list: NestedTensor): + x = tensor_list.tensors + mask = tensor_list.mask + assert mask is not None + not_mask = ~mask + y_embed = not_mask.cumsum(1, dtype=torch.float32) + x_embed = not_mask.cumsum(2, dtype=torch.float32) + if self.normalize: + eps = 1e-6 + # if os.environ.get("SHILONG_AMP", None) == '1': + # eps = 1e-4 + # else: + # eps = 1e-6 + y_embed = y_embed / (y_embed[:, -1:, :] + eps) * self.scale + x_embed = x_embed / (x_embed[:, :, -1:] + eps) * self.scale + + dim_t = torch.arange(self.num_pos_feats, dtype=torch.float32, device=x.device) + dim_t = self.temperature ** (2 * (dim_t // 2) / self.num_pos_feats) + + pos_x = x_embed[:, :, :, None] / dim_t + pos_y = y_embed[:, :, :, None] / dim_t + pos_x = torch.stack( + (pos_x[:, :, :, 0::2].sin(), pos_x[:, :, :, 1::2].cos()), dim=4 + ).flatten(3) + pos_y = torch.stack( + (pos_y[:, :, :, 0::2].sin(), pos_y[:, :, :, 1::2].cos()), dim=4 + ).flatten(3) + pos = torch.cat((pos_y, pos_x), dim=3).permute(0, 3, 1, 2) + return pos + + +class PositionEmbeddingSineHW(nn.Module): + """ + This is a more standard version of the position embedding, very similar to the one + used by the Attention is all you need paper, generalized to work on images. + """ + + def __init__( + self, num_pos_feats=64, temperatureH=10000, temperatureW=10000, normalize=False, scale=None + ): + super().__init__() + self.num_pos_feats = num_pos_feats + self.temperatureH = temperatureH + self.temperatureW = temperatureW + self.normalize = normalize + if scale is not None and normalize is False: + raise ValueError("normalize should be True if scale is passed") + if scale is None: + scale = 2 * math.pi + self.scale = scale + + def forward(self, tensor_list: NestedTensor): + x = tensor_list.tensors + mask = tensor_list.mask + assert mask is not None + not_mask = ~mask + y_embed = not_mask.cumsum(1, dtype=torch.float32) + x_embed = not_mask.cumsum(2, dtype=torch.float32) + + # import ipdb; ipdb.set_trace() + + if self.normalize: + eps = 1e-6 + y_embed = y_embed / (y_embed[:, -1:, :] + eps) * self.scale + x_embed = x_embed / (x_embed[:, :, -1:] + eps) * self.scale + + dim_tx = torch.arange(self.num_pos_feats, dtype=torch.float32, device=x.device) + dim_tx = self.temperatureW ** (2 * (torch.div(dim_tx, 2, rounding_mode='floor')) / self.num_pos_feats) + pos_x = x_embed[:, :, :, None] / dim_tx + + dim_ty = torch.arange(self.num_pos_feats, dtype=torch.float32, device=x.device) + dim_ty = self.temperatureH ** (2 * (torch.div(dim_ty, 2, rounding_mode='floor')) / self.num_pos_feats) + pos_y = y_embed[:, :, :, None] / dim_ty + + pos_x = torch.stack( + (pos_x[:, :, :, 0::2].sin(), pos_x[:, :, :, 1::2].cos()), dim=4 + ).flatten(3) + pos_y = torch.stack( + (pos_y[:, :, :, 0::2].sin(), pos_y[:, :, :, 1::2].cos()), dim=4 + ).flatten(3) + pos = torch.cat((pos_y, pos_x), dim=3).permute(0, 3, 1, 2) + + # import ipdb; ipdb.set_trace() + + return pos + + +class PositionEmbeddingLearned(nn.Module): + """ + Absolute pos embedding, learned. + """ + + def __init__(self, num_pos_feats=256): + super().__init__() + self.row_embed = nn.Embedding(50, num_pos_feats) + self.col_embed = nn.Embedding(50, num_pos_feats) + self.reset_parameters() + + def reset_parameters(self): + nn.init.uniform_(self.row_embed.weight) + nn.init.uniform_(self.col_embed.weight) + + def forward(self, tensor_list: NestedTensor): + x = tensor_list.tensors + h, w = x.shape[-2:] + i = torch.arange(w, device=x.device) + j = torch.arange(h, device=x.device) + x_emb = self.col_embed(i) + y_emb = self.row_embed(j) + pos = ( + torch.cat( + [ + x_emb.unsqueeze(0).repeat(h, 1, 1), + y_emb.unsqueeze(1).repeat(1, w, 1), + ], + dim=-1, + ) + .permute(2, 0, 1) + .unsqueeze(0) + .repeat(x.shape[0], 1, 1, 1) + ) + return pos + + +def build_position_encoding(args): + N_steps = args.hidden_dim // 2 + if args.position_embedding in ("v2", "sine"): + # TODO find a better way of exposing other arguments + position_embedding = PositionEmbeddingSineHW( + N_steps, + temperatureH=args.pe_temperatureH, + temperatureW=args.pe_temperatureW, + normalize=True, + ) + elif args.position_embedding in ("v3", "learned"): + position_embedding = PositionEmbeddingLearned(N_steps) + else: + raise ValueError(f"not supported {args.position_embedding}") + + return position_embedding diff --git a/models/GroundingDINO/backbone/swin_transformer.py b/models/GroundingDINO/backbone/swin_transformer.py new file mode 100644 index 0000000000000000000000000000000000000000..affebbb7356ae2c01e23fa852d7105aab5b3a8f4 --- /dev/null +++ b/models/GroundingDINO/backbone/swin_transformer.py @@ -0,0 +1,804 @@ +# ------------------------------------------------------------------------ +# Grounding DINO +# url: https://github.com/IDEA-Research/GroundingDINO +# Copyright (c) 2023 IDEA. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------ +# DINO +# Copyright (c) 2022 IDEA. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# -------------------------------------------------------- +# modified from https://github.com/SwinTransformer/Swin-Transformer-Object-Detection/blob/master/mmdet/models/backbones/swin_transformer.py +# -------------------------------------------------------- + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.utils.checkpoint as checkpoint +from timm.models.layers import DropPath, to_2tuple, trunc_normal_ + +from groundingdino.util.misc import NestedTensor + + +class Mlp(nn.Module): + """Multilayer perceptron.""" + + def __init__( + self, in_features, hidden_features=None, out_features=None, act_layer=nn.GELU, drop=0.0 + ): + super().__init__() + out_features = out_features or in_features + hidden_features = hidden_features or in_features + self.fc1 = nn.Linear(in_features, hidden_features) + self.act = act_layer() + self.fc2 = nn.Linear(hidden_features, out_features) + self.drop = nn.Dropout(drop) + + def forward(self, x): + x = self.fc1(x) + x = self.act(x) + x = self.drop(x) + x = self.fc2(x) + x = self.drop(x) + return x + + +def window_partition(x, window_size): + """ + Args: + x: (B, H, W, C) + window_size (int): window size + Returns: + windows: (num_windows*B, window_size, window_size, C) + """ + B, H, W, C = x.shape + x = x.view(B, H // window_size, window_size, W // window_size, window_size, C) + windows = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(-1, window_size, window_size, C) + return windows + + +def window_reverse(windows, window_size, H, W): + """ + Args: + windows: (num_windows*B, window_size, window_size, C) + window_size (int): Window size + H (int): Height of image + W (int): Width of image + Returns: + x: (B, H, W, C) + """ + B = int(windows.shape[0] / (H * W / window_size / window_size)) + x = windows.view(B, H // window_size, W // window_size, window_size, window_size, -1) + x = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(B, H, W, -1) + return x + + +class WindowAttention(nn.Module): + """Window based multi-head self attention (W-MSA) module with relative position bias. + It supports both of shifted and non-shifted window. + Args: + dim (int): Number of input channels. + window_size (tuple[int]): The height and width of the window. + num_heads (int): Number of attention heads. + qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True + qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set + attn_drop (float, optional): Dropout ratio of attention weight. Default: 0.0 + proj_drop (float, optional): Dropout ratio of output. Default: 0.0 + """ + + def __init__( + self, + dim, + window_size, + num_heads, + qkv_bias=True, + qk_scale=None, + attn_drop=0.0, + proj_drop=0.0, + ): + + super().__init__() + self.dim = dim + self.window_size = window_size # Wh, Ww + self.num_heads = num_heads + head_dim = dim // num_heads + self.scale = qk_scale or head_dim**-0.5 + + # define a parameter table of relative position bias + self.relative_position_bias_table = nn.Parameter( + torch.zeros((2 * window_size[0] - 1) * (2 * window_size[1] - 1), num_heads) + ) # 2*Wh-1 * 2*Ww-1, nH + + # get pair-wise relative position index for each token inside the window + coords_h = torch.arange(self.window_size[0]) + coords_w = torch.arange(self.window_size[1]) + coords = torch.stack(torch.meshgrid([coords_h, coords_w])) # 2, Wh, Ww + coords_flatten = torch.flatten(coords, 1) # 2, Wh*Ww + relative_coords = coords_flatten[:, :, None] - coords_flatten[:, None, :] # 2, Wh*Ww, Wh*Ww + relative_coords = relative_coords.permute(1, 2, 0).contiguous() # Wh*Ww, Wh*Ww, 2 + relative_coords[:, :, 0] += self.window_size[0] - 1 # shift to start from 0 + relative_coords[:, :, 1] += self.window_size[1] - 1 + relative_coords[:, :, 0] *= 2 * self.window_size[1] - 1 + relative_position_index = relative_coords.sum(-1) # Wh*Ww, Wh*Ww + self.register_buffer("relative_position_index", relative_position_index) + + self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias) + self.attn_drop = nn.Dropout(attn_drop) + self.proj = nn.Linear(dim, dim) + self.proj_drop = nn.Dropout(proj_drop) + + trunc_normal_(self.relative_position_bias_table, std=0.02) + self.softmax = nn.Softmax(dim=-1) + + def forward(self, x, mask=None): + """Forward function. + Args: + x: input features with shape of (num_windows*B, N, C) + mask: (0/-inf) mask with shape of (num_windows, Wh*Ww, Wh*Ww) or None + """ + B_, N, C = x.shape + qkv = ( + self.qkv(x) + .reshape(B_, N, 3, self.num_heads, C // self.num_heads) + .permute(2, 0, 3, 1, 4) + ) + q, k, v = qkv[0], qkv[1], qkv[2] # make torchscript happy (cannot use tensor as tuple) + + q = q * self.scale + attn = q @ k.transpose(-2, -1) + + relative_position_bias = self.relative_position_bias_table[ + self.relative_position_index.view(-1) + ].view( + self.window_size[0] * self.window_size[1], self.window_size[0] * self.window_size[1], -1 + ) # Wh*Ww,Wh*Ww,nH + relative_position_bias = relative_position_bias.permute( + 2, 0, 1 + ).contiguous() # nH, Wh*Ww, Wh*Ww + attn = attn + relative_position_bias.unsqueeze(0) + + if mask is not None: + nW = mask.shape[0] + attn = attn.view(B_ // nW, nW, self.num_heads, N, N) + mask.unsqueeze(1).unsqueeze(0) + attn = attn.view(-1, self.num_heads, N, N) + attn = self.softmax(attn) + else: + attn = self.softmax(attn) + + attn = self.attn_drop(attn) + + x = (attn @ v).transpose(1, 2).reshape(B_, N, C) + x = self.proj(x) + x = self.proj_drop(x) + return x + + +class SwinTransformerBlock(nn.Module): + """Swin Transformer Block. + Args: + dim (int): Number of input channels. + num_heads (int): Number of attention heads. + window_size (int): Window size. + shift_size (int): Shift size for SW-MSA. + mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. + qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True + qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set. + drop (float, optional): Dropout rate. Default: 0.0 + attn_drop (float, optional): Attention dropout rate. Default: 0.0 + drop_path (float, optional): Stochastic depth rate. Default: 0.0 + act_layer (nn.Module, optional): Activation layer. Default: nn.GELU + norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm + """ + + def __init__( + self, + dim, + num_heads, + window_size=7, + shift_size=0, + mlp_ratio=4.0, + qkv_bias=True, + qk_scale=None, + drop=0.0, + attn_drop=0.0, + drop_path=0.0, + act_layer=nn.GELU, + norm_layer=nn.LayerNorm, + ): + super().__init__() + self.dim = dim + self.num_heads = num_heads + self.window_size = window_size + self.shift_size = shift_size + self.mlp_ratio = mlp_ratio + assert 0 <= self.shift_size < self.window_size, "shift_size must in 0-window_size" + + self.norm1 = norm_layer(dim) + self.attn = WindowAttention( + dim, + window_size=to_2tuple(self.window_size), + num_heads=num_heads, + qkv_bias=qkv_bias, + qk_scale=qk_scale, + attn_drop=attn_drop, + proj_drop=drop, + ) + + self.drop_path = DropPath(drop_path) if drop_path > 0.0 else nn.Identity() + self.norm2 = norm_layer(dim) + mlp_hidden_dim = int(dim * mlp_ratio) + self.mlp = Mlp( + in_features=dim, hidden_features=mlp_hidden_dim, act_layer=act_layer, drop=drop + ) + + self.H = None + self.W = None + + def forward(self, x, mask_matrix): + """Forward function. + Args: + x: Input feature, tensor size (B, H*W, C). + H, W: Spatial resolution of the input feature. + mask_matrix: Attention mask for cyclic shift. + """ + B, L, C = x.shape + H, W = self.H, self.W + assert L == H * W, "input feature has wrong size" + + shortcut = x + x = self.norm1(x) + x = x.view(B, H, W, C) + + # pad feature maps to multiples of window size + pad_l = pad_t = 0 + pad_r = (self.window_size - W % self.window_size) % self.window_size + pad_b = (self.window_size - H % self.window_size) % self.window_size + x = F.pad(x, (0, 0, pad_l, pad_r, pad_t, pad_b)) + _, Hp, Wp, _ = x.shape + + # cyclic shift + if self.shift_size > 0: + shifted_x = torch.roll(x, shifts=(-self.shift_size, -self.shift_size), dims=(1, 2)) + attn_mask = mask_matrix + else: + shifted_x = x + attn_mask = None + + # partition windows + x_windows = window_partition( + shifted_x, self.window_size + ) # nW*B, window_size, window_size, C + x_windows = x_windows.view( + -1, self.window_size * self.window_size, C + ) # nW*B, window_size*window_size, C + + # W-MSA/SW-MSA + attn_windows = self.attn(x_windows, mask=attn_mask) # nW*B, window_size*window_size, C + + # merge windows + attn_windows = attn_windows.view(-1, self.window_size, self.window_size, C) + shifted_x = window_reverse(attn_windows, self.window_size, Hp, Wp) # B H' W' C + + # reverse cyclic shift + if self.shift_size > 0: + x = torch.roll(shifted_x, shifts=(self.shift_size, self.shift_size), dims=(1, 2)) + else: + x = shifted_x + + if pad_r > 0 or pad_b > 0: + x = x[:, :H, :W, :].contiguous() + + x = x.view(B, H * W, C) + + # FFN + x = shortcut + self.drop_path(x) + x = x + self.drop_path(self.mlp(self.norm2(x))) + + return x + + +class PatchMerging(nn.Module): + """Patch Merging Layer + Args: + dim (int): Number of input channels. + norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm + """ + + def __init__(self, dim, norm_layer=nn.LayerNorm): + super().__init__() + self.dim = dim + self.reduction = nn.Linear(4 * dim, 2 * dim, bias=False) + self.norm = norm_layer(4 * dim) + + def forward(self, x, H, W): + """Forward function. + Args: + x: Input feature, tensor size (B, H*W, C). + H, W: Spatial resolution of the input feature. + """ + B, L, C = x.shape + assert L == H * W, "input feature has wrong size" + + x = x.view(B, H, W, C) + + # padding + pad_input = (H % 2 == 1) or (W % 2 == 1) + if pad_input: + x = F.pad(x, (0, 0, 0, W % 2, 0, H % 2)) + + x0 = x[:, 0::2, 0::2, :] # B H/2 W/2 C + x1 = x[:, 1::2, 0::2, :] # B H/2 W/2 C + x2 = x[:, 0::2, 1::2, :] # B H/2 W/2 C + x3 = x[:, 1::2, 1::2, :] # B H/2 W/2 C + x = torch.cat([x0, x1, x2, x3], -1) # B H/2 W/2 4*C + x = x.view(B, -1, 4 * C) # B H/2*W/2 4*C + + x = self.norm(x) + x = self.reduction(x) + + return x + + +class BasicLayer(nn.Module): + """A basic Swin Transformer layer for one stage. + Args: + dim (int): Number of feature channels + depth (int): Depths of this stage. + num_heads (int): Number of attention head. + window_size (int): Local window size. Default: 7. + mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. Default: 4. + qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True + qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set. + drop (float, optional): Dropout rate. Default: 0.0 + attn_drop (float, optional): Attention dropout rate. Default: 0.0 + drop_path (float | tuple[float], optional): Stochastic depth rate. Default: 0.0 + norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm + downsample (nn.Module | None, optional): Downsample layer at the end of the layer. Default: None + use_checkpoint (bool): Whether to use checkpointing to save memory. Default: False. + """ + + def __init__( + self, + dim, + depth, + num_heads, + window_size=7, + mlp_ratio=4.0, + qkv_bias=True, + qk_scale=None, + drop=0.0, + attn_drop=0.0, + drop_path=0.0, + norm_layer=nn.LayerNorm, + downsample=None, + use_checkpoint=False, + ): + super().__init__() + self.window_size = window_size + self.shift_size = window_size // 2 + self.depth = depth + self.use_checkpoint = use_checkpoint + + # build blocks + self.blocks = nn.ModuleList( + [ + SwinTransformerBlock( + dim=dim, + num_heads=num_heads, + window_size=window_size, + shift_size=0 if (i % 2 == 0) else window_size // 2, + mlp_ratio=mlp_ratio, + qkv_bias=qkv_bias, + qk_scale=qk_scale, + drop=drop, + attn_drop=attn_drop, + drop_path=drop_path[i] if isinstance(drop_path, list) else drop_path, + norm_layer=norm_layer, + ) + for i in range(depth) + ] + ) + + # patch merging layer + if downsample is not None: + self.downsample = downsample(dim=dim, norm_layer=norm_layer) + else: + self.downsample = None + + def forward(self, x, H, W): + """Forward function. + Args: + x: Input feature, tensor size (B, H*W, C). + H, W: Spatial resolution of the input feature. + """ + + # calculate attention mask for SW-MSA + Hp = int(np.ceil(H / self.window_size)) * self.window_size + Wp = int(np.ceil(W / self.window_size)) * self.window_size + img_mask = torch.zeros((1, Hp, Wp, 1), device=x.device) # 1 Hp Wp 1 + h_slices = ( + slice(0, -self.window_size), + slice(-self.window_size, -self.shift_size), + slice(-self.shift_size, None), + ) + w_slices = ( + slice(0, -self.window_size), + slice(-self.window_size, -self.shift_size), + slice(-self.shift_size, None), + ) + cnt = 0 + for h in h_slices: + for w in w_slices: + img_mask[:, h, w, :] = cnt + cnt += 1 + + mask_windows = window_partition( + img_mask, self.window_size + ) # nW, window_size, window_size, 1 + mask_windows = mask_windows.view(-1, self.window_size * self.window_size) + attn_mask = mask_windows.unsqueeze(1) - mask_windows.unsqueeze(2) + attn_mask = attn_mask.masked_fill(attn_mask != 0, float(-100.0)).masked_fill( + attn_mask == 0, float(0.0) + ) + + for blk in self.blocks: + blk.H, blk.W = H, W + if self.use_checkpoint: + x = checkpoint.checkpoint(blk, x, attn_mask) + else: + x = blk(x, attn_mask) + if self.downsample is not None: + x_down = self.downsample(x, H, W) + Wh, Ww = (H + 1) // 2, (W + 1) // 2 + return x, H, W, x_down, Wh, Ww + else: + return x, H, W, x, H, W + + +class PatchEmbed(nn.Module): + """Image to Patch Embedding + Args: + patch_size (int): Patch token size. Default: 4. + in_chans (int): Number of input image channels. Default: 3. + embed_dim (int): Number of linear projection output channels. Default: 96. + norm_layer (nn.Module, optional): Normalization layer. Default: None + """ + + def __init__(self, patch_size=4, in_chans=3, embed_dim=96, norm_layer=None): + super().__init__() + patch_size = to_2tuple(patch_size) + self.patch_size = patch_size + + self.in_chans = in_chans + self.embed_dim = embed_dim + + self.proj = nn.Conv2d(in_chans, embed_dim, kernel_size=patch_size, stride=patch_size) + if norm_layer is not None: + self.norm = norm_layer(embed_dim) + else: + self.norm = None + + def forward(self, x): + """Forward function.""" + # padding + _, _, H, W = x.size() + if W % self.patch_size[1] != 0: + x = F.pad(x, (0, self.patch_size[1] - W % self.patch_size[1])) + if H % self.patch_size[0] != 0: + x = F.pad(x, (0, 0, 0, self.patch_size[0] - H % self.patch_size[0])) + + x = self.proj(x) # B C Wh Ww + if self.norm is not None: + Wh, Ww = x.size(2), x.size(3) + x = x.flatten(2).transpose(1, 2) + x = self.norm(x) + x = x.transpose(1, 2).view(-1, self.embed_dim, Wh, Ww) + + return x + + +class SwinTransformer(nn.Module): + """Swin Transformer backbone. + A PyTorch impl of : `Swin Transformer: Hierarchical Vision Transformer using Shifted Windows` - + https://arxiv.org/pdf/2103.14030 + Args: + pretrain_img_size (int): Input image size for training the pretrained model, + used in absolute postion embedding. Default 224. + patch_size (int | tuple(int)): Patch size. Default: 4. + in_chans (int): Number of input image channels. Default: 3. + embed_dim (int): Number of linear projection output channels. Default: 96. + depths (tuple[int]): Depths of each Swin Transformer stage. + num_heads (tuple[int]): Number of attention head of each stage. + window_size (int): Window size. Default: 7. + mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. Default: 4. + qkv_bias (bool): If True, add a learnable bias to query, key, value. Default: True + qk_scale (float): Override default qk scale of head_dim ** -0.5 if set. + drop_rate (float): Dropout rate. + attn_drop_rate (float): Attention dropout rate. Default: 0. + drop_path_rate (float): Stochastic depth rate. Default: 0.2. + norm_layer (nn.Module): Normalization layer. Default: nn.LayerNorm. + ape (bool): If True, add absolute position embedding to the patch embedding. Default: False. + patch_norm (bool): If True, add normalization after patch embedding. Default: True. + out_indices (Sequence[int]): Output from which stages. + frozen_stages (int): Stages to be frozen (stop grad and set eval mode). + -1 means not freezing any parameters. + use_checkpoint (bool): Whether to use checkpointing to save memory. Default: False. + dilation (bool): if True, the output size if 16x downsample, ow 32x downsample. + """ + + def __init__( + self, + pretrain_img_size=224, + patch_size=4, + in_chans=3, + embed_dim=96, + depths=[2, 2, 6, 2], + num_heads=[3, 6, 12, 24], + window_size=7, + mlp_ratio=4.0, + qkv_bias=True, + qk_scale=None, + drop_rate=0.0, + attn_drop_rate=0.0, + drop_path_rate=0.2, + norm_layer=nn.LayerNorm, + ape=False, + patch_norm=True, + out_indices=(0, 1, 2, 3), + frozen_stages=-1, + dilation=False, + use_checkpoint=False, + ): + super().__init__() + + self.pretrain_img_size = pretrain_img_size + self.num_layers = len(depths) + self.embed_dim = embed_dim + self.ape = ape + self.patch_norm = patch_norm + self.out_indices = out_indices + self.frozen_stages = frozen_stages + self.dilation = dilation + + # if use_checkpoint: + # print("use_checkpoint!!!!!!!!!!!!!!!!!!!!!!!!") + + # split image into non-overlapping patches + self.patch_embed = PatchEmbed( + patch_size=patch_size, + in_chans=in_chans, + embed_dim=embed_dim, + norm_layer=norm_layer if self.patch_norm else None, + ) + + # absolute position embedding + if self.ape: + pretrain_img_size = to_2tuple(pretrain_img_size) + patch_size = to_2tuple(patch_size) + patches_resolution = [ + pretrain_img_size[0] // patch_size[0], + pretrain_img_size[1] // patch_size[1], + ] + + self.absolute_pos_embed = nn.Parameter( + torch.zeros(1, embed_dim, patches_resolution[0], patches_resolution[1]) + ) + trunc_normal_(self.absolute_pos_embed, std=0.02) + + self.pos_drop = nn.Dropout(p=drop_rate) + + # stochastic depth + dpr = [ + x.item() for x in torch.linspace(0, drop_path_rate, sum(depths)) + ] # stochastic depth decay rule + + # build layers + self.layers = nn.ModuleList() + # prepare downsample list + downsamplelist = [PatchMerging for i in range(self.num_layers)] + downsamplelist[-1] = None + num_features = [int(embed_dim * 2**i) for i in range(self.num_layers)] + if self.dilation: + downsamplelist[-2] = None + num_features[-1] = int(embed_dim * 2 ** (self.num_layers - 1)) // 2 + for i_layer in range(self.num_layers): + layer = BasicLayer( + # dim=int(embed_dim * 2 ** i_layer), + dim=num_features[i_layer], + depth=depths[i_layer], + num_heads=num_heads[i_layer], + window_size=window_size, + mlp_ratio=mlp_ratio, + qkv_bias=qkv_bias, + qk_scale=qk_scale, + drop=drop_rate, + attn_drop=attn_drop_rate, + drop_path=dpr[sum(depths[:i_layer]) : sum(depths[: i_layer + 1])], + norm_layer=norm_layer, + # downsample=PatchMerging if (i_layer < self.num_layers - 1) else None, + downsample=downsamplelist[i_layer], + use_checkpoint=use_checkpoint, + ) + self.layers.append(layer) + + # num_features = [int(embed_dim * 2 ** i) for i in range(self.num_layers)] + self.num_features = num_features + + # add a norm layer for each output + for i_layer in out_indices: + layer = norm_layer(num_features[i_layer]) + layer_name = f"norm{i_layer}" + self.add_module(layer_name, layer) + + self._freeze_stages() + + def _freeze_stages(self): + if self.frozen_stages >= 0: + self.patch_embed.eval() + for param in self.patch_embed.parameters(): + param.requires_grad = False + + if self.frozen_stages >= 1 and self.ape: + self.absolute_pos_embed.requires_grad = False + + if self.frozen_stages >= 2: + self.pos_drop.eval() + for i in range(0, self.frozen_stages - 1): + m = self.layers[i] + m.eval() + for param in m.parameters(): + param.requires_grad = False + + # def init_weights(self, pretrained=None): + # """Initialize the weights in backbone. + # Args: + # pretrained (str, optional): Path to pre-trained weights. + # Defaults to None. + # """ + + # def _init_weights(m): + # if isinstance(m, nn.Linear): + # trunc_normal_(m.weight, std=.02) + # if isinstance(m, nn.Linear) and m.bias is not None: + # nn.init.constant_(m.bias, 0) + # elif isinstance(m, nn.LayerNorm): + # nn.init.constant_(m.bias, 0) + # nn.init.constant_(m.weight, 1.0) + + # if isinstance(pretrained, str): + # self.apply(_init_weights) + # logger = get_root_logger() + # load_checkpoint(self, pretrained, strict=False, logger=logger) + # elif pretrained is None: + # self.apply(_init_weights) + # else: + # raise TypeError('pretrained must be a str or None') + + def forward_raw(self, x): + """Forward function.""" + x = self.patch_embed(x) + + Wh, Ww = x.size(2), x.size(3) + if self.ape: + # interpolate the position embedding to the corresponding size + absolute_pos_embed = F.interpolate( + self.absolute_pos_embed, size=(Wh, Ww), mode="bicubic" + ) + x = (x + absolute_pos_embed).flatten(2).transpose(1, 2) # B Wh*Ww C + else: + x = x.flatten(2).transpose(1, 2) + x = self.pos_drop(x) + + outs = [] + for i in range(self.num_layers): + layer = self.layers[i] + x_out, H, W, x, Wh, Ww = layer(x, Wh, Ww) + # import ipdb; ipdb.set_trace() + + if i in self.out_indices: + norm_layer = getattr(self, f"norm{i}") + x_out = norm_layer(x_out) + + out = x_out.view(-1, H, W, self.num_features[i]).permute(0, 3, 1, 2).contiguous() + outs.append(out) + # in: + # torch.Size([2, 3, 1024, 1024]) + # outs: + # [torch.Size([2, 192, 256, 256]), torch.Size([2, 384, 128, 128]), \ + # torch.Size([2, 768, 64, 64]), torch.Size([2, 1536, 32, 32])] + return tuple(outs) + + def forward(self, tensor_list: NestedTensor): + + x = tensor_list.tensors + + """Forward function.""" + x = self.patch_embed(x) + + + Wh, Ww = x.size(2), x.size(3) + if self.ape: + # interpolate the position embedding to the corresponding size + absolute_pos_embed = F.interpolate( + self.absolute_pos_embed, size=(Wh, Ww), mode="bicubic" + ) + x = (x + absolute_pos_embed).flatten(2).transpose(1, 2) # B Wh*Ww C + else: + x = x.flatten(2).transpose(1, 2) + x = self.pos_drop(x) + + outs = [] + for i in range(self.num_layers): + layer = self.layers[i] + x_out, H, W, x, Wh, Ww = layer(x, Wh, Ww) + + if i in self.out_indices: + norm_layer = getattr(self, f"norm{i}") + x_out = norm_layer(x_out) + + out = x_out.view(-1, H, W, self.num_features[i]).permute(0, 3, 1, 2).contiguous() + outs.append(out) + # in: + # torch.Size([2, 3, 1024, 1024]) + # out: + # [torch.Size([2, 192, 256, 256]), torch.Size([2, 384, 128, 128]), \ + # torch.Size([2, 768, 64, 64]), torch.Size([2, 1536, 32, 32])] + + # collect for nesttensors + outs_dict = {} + for idx, out_i in enumerate(outs): + m = tensor_list.mask + assert m is not None + mask = F.interpolate(m[None].float(), size=out_i.shape[-2:]).to(torch.bool)[0] + outs_dict[idx] = NestedTensor(out_i, mask) + + return outs_dict + + def train(self, mode=True): + """Convert the model into training mode while keep layers freezed.""" + super(SwinTransformer, self).train(mode) + self._freeze_stages() + + +def build_swin_transformer(modelname, pretrain_img_size, **kw): + assert modelname in [ + "swin_T_224_1k", + "swin_B_224_22k", + "swin_B_384_22k", + "swin_L_224_22k", + "swin_L_384_22k", + ] + + model_para_dict = { + "swin_T_224_1k": dict( + embed_dim=96, depths=[2, 2, 6, 2], num_heads=[3, 6, 12, 24], window_size=7 + ), + "swin_B_224_22k": dict( + embed_dim=128, depths=[2, 2, 18, 2], num_heads=[4, 8, 16, 32], window_size=7 + ), + "swin_B_384_22k": dict( + embed_dim=128, depths=[2, 2, 18, 2], num_heads=[4, 8, 16, 32], window_size=12 + ), + "swin_L_224_22k": dict( + embed_dim=192, depths=[2, 2, 18, 2], num_heads=[6, 12, 24, 48], window_size=7 + ), + "swin_L_384_22k": dict( + embed_dim=192, depths=[2, 2, 18, 2], num_heads=[6, 12, 24, 48], window_size=12 + ), + } + kw_cgf = model_para_dict[modelname] + kw_cgf.update(kw) + model = SwinTransformer(pretrain_img_size=pretrain_img_size, **kw_cgf) + return model + + +if __name__ == "__main__": + model = build_swin_transformer("swin_L_384_22k", 384, dilation=True) + x = torch.rand(2, 3, 1024, 1024) + y = model.forward_raw(x) + import ipdb + + ipdb.set_trace() + x = torch.rand(2, 3, 384, 384) + y = model.forward_raw(x) diff --git a/models/GroundingDINO/bertwarper.py b/models/GroundingDINO/bertwarper.py new file mode 100644 index 0000000000000000000000000000000000000000..f0cf9779b270e1aead32845006f8b881fcba37ad --- /dev/null +++ b/models/GroundingDINO/bertwarper.py @@ -0,0 +1,273 @@ +# ------------------------------------------------------------------------ +# Grounding DINO +# url: https://github.com/IDEA-Research/GroundingDINO +# Copyright (c) 2023 IDEA. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------ + +import torch +import torch.nn.functional as F +import torch.utils.checkpoint as checkpoint +from torch import Tensor, nn +from torchvision.ops.boxes import nms +from transformers import BertConfig, BertModel, BertPreTrainedModel +from transformers.modeling_outputs import BaseModelOutputWithPoolingAndCrossAttentions + + +class BertModelWarper(nn.Module): + def __init__(self, bert_model): + super().__init__() + # self.bert = bert_modelc + + self.config = bert_model.config + self.embeddings = bert_model.embeddings + self.encoder = bert_model.encoder + self.pooler = bert_model.pooler + + self.get_extended_attention_mask = bert_model.get_extended_attention_mask + self.invert_attention_mask = bert_model.invert_attention_mask + self.get_head_mask = bert_model.get_head_mask + + def forward( + self, + input_ids=None, + attention_mask=None, + token_type_ids=None, + position_ids=None, + head_mask=None, + inputs_embeds=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + past_key_values=None, + use_cache=None, + output_attentions=None, + output_hidden_states=None, + return_dict=None, + ): + r""" + encoder_hidden_states (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length, hidden_size)`, `optional`): + Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if + the model is configured as a decoder. + encoder_attention_mask (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`): + Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in + the cross-attention if the model is configured as a decoder. Mask values selected in ``[0, 1]``: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + past_key_values (:obj:`tuple(tuple(torch.FloatTensor))` of length :obj:`config.n_layers` with each tuple having 4 tensors of shape :obj:`(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): + Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding. + + If :obj:`past_key_values` are used, the user can optionally input only the last :obj:`decoder_input_ids` + (those that don't have their past key value states given to this model) of shape :obj:`(batch_size, 1)` + instead of all :obj:`decoder_input_ids` of shape :obj:`(batch_size, sequence_length)`. + use_cache (:obj:`bool`, `optional`): + If set to :obj:`True`, :obj:`past_key_values` key value states are returned and can be used to speed up + decoding (see :obj:`past_key_values`). + """ + output_attentions = ( + output_attentions if output_attentions is not None else self.config.output_attentions + ) + output_hidden_states = ( + output_hidden_states + if output_hidden_states is not None + else self.config.output_hidden_states + ) + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + if self.config.is_decoder: + use_cache = use_cache if use_cache is not None else self.config.use_cache + else: + use_cache = False + + if input_ids is not None and inputs_embeds is not None: + raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time") + elif input_ids is not None: + input_shape = input_ids.size() + batch_size, seq_length = input_shape + elif inputs_embeds is not None: + input_shape = inputs_embeds.size()[:-1] + batch_size, seq_length = input_shape + else: + raise ValueError("You have to specify either input_ids or inputs_embeds") + + device = input_ids.device if input_ids is not None else inputs_embeds.device + + # past_key_values_length + past_key_values_length = ( + past_key_values[0][0].shape[2] if past_key_values is not None else 0 + ) + + if attention_mask is None: + attention_mask = torch.ones( + ((batch_size, seq_length + past_key_values_length)), device=device + ) + if token_type_ids is None: + token_type_ids = torch.zeros(input_shape, dtype=torch.long, device=device) + + # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length] + # ourselves in which case we just need to make it broadcastable to all heads. + extended_attention_mask: torch.Tensor = self.get_extended_attention_mask( + attention_mask, input_shape, device + ) + + # If a 2D or 3D attention mask is provided for the cross-attention + # we need to make broadcastable to [batch_size, num_heads, seq_length, seq_length] + if self.config.is_decoder and encoder_hidden_states is not None: + encoder_batch_size, encoder_sequence_length, _ = encoder_hidden_states.size() + encoder_hidden_shape = (encoder_batch_size, encoder_sequence_length) + if encoder_attention_mask is None: + encoder_attention_mask = torch.ones(encoder_hidden_shape, device=device) + encoder_extended_attention_mask = self.invert_attention_mask(encoder_attention_mask) + else: + encoder_extended_attention_mask = None + # if os.environ.get('IPDB_SHILONG_DEBUG', None) == 'INFO': + # import ipdb; ipdb.set_trace() + + # Prepare head mask if needed + # 1.0 in head_mask indicate we keep the head + # attention_probs has shape bsz x n_heads x N x N + # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads] + # and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length] + head_mask = self.get_head_mask(head_mask, self.config.num_hidden_layers) + + embedding_output = self.embeddings( + input_ids=input_ids, + position_ids=position_ids, + token_type_ids=token_type_ids, + inputs_embeds=inputs_embeds, + past_key_values_length=past_key_values_length, + ) + + encoder_outputs = self.encoder( + embedding_output, + attention_mask=extended_attention_mask, + head_mask=head_mask, + encoder_hidden_states=encoder_hidden_states, + encoder_attention_mask=encoder_extended_attention_mask, + past_key_values=past_key_values, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + sequence_output = encoder_outputs[0] + pooled_output = self.pooler(sequence_output) if self.pooler is not None else None + + if not return_dict: + return (sequence_output, pooled_output) + encoder_outputs[1:] + + return BaseModelOutputWithPoolingAndCrossAttentions( + last_hidden_state=sequence_output, + pooler_output=pooled_output, + past_key_values=encoder_outputs.past_key_values, + hidden_states=encoder_outputs.hidden_states, + attentions=encoder_outputs.attentions, + cross_attentions=encoder_outputs.cross_attentions, + ) + + +class TextEncoderShell(nn.Module): + def __init__(self, text_encoder): + super().__init__() + self.text_encoder = text_encoder + self.config = self.text_encoder.config + + def forward(self, **kw): + # feed into text encoder + return self.text_encoder(**kw) + + +def generate_masks_with_special_tokens(tokenized, special_tokens_list, tokenizer): + """Generate attention mask between each pair of special tokens + Args: + input_ids (torch.Tensor): input ids. Shape: [bs, num_token] + special_tokens_mask (list): special tokens mask. + Returns: + torch.Tensor: attention mask between each special tokens. + """ + input_ids = tokenized["input_ids"] + bs, num_token = input_ids.shape + # special_tokens_mask: bs, num_token. 1 for special tokens. 0 for normal tokens + special_tokens_mask = torch.zeros((bs, num_token), device=input_ids.device).bool() + for special_token in special_tokens_list: + special_tokens_mask |= input_ids == special_token + + # idxs: each row is a list of indices of special tokens + idxs = torch.nonzero(special_tokens_mask) + + # generate attention mask and positional ids + attention_mask = ( + torch.eye(num_token, device=input_ids.device).bool().unsqueeze(0).repeat(bs, 1, 1) + ) + position_ids = torch.zeros((bs, num_token), device=input_ids.device) + previous_col = 0 + for i in range(idxs.shape[0]): + row, col = idxs[i] + if (col == 0) or (col == num_token - 1): + attention_mask[row, col, col] = True + position_ids[row, col] = 0 + else: + attention_mask[row, previous_col + 1 : col + 1, previous_col + 1 : col + 1] = True + position_ids[row, previous_col + 1 : col + 1] = torch.arange( + 0, col - previous_col, device=input_ids.device + ) + + previous_col = col + + # # padding mask + # padding_mask = tokenized['attention_mask'] + # attention_mask = attention_mask & padding_mask.unsqueeze(1).bool() & padding_mask.unsqueeze(2).bool() + + return attention_mask, position_ids.to(torch.long) + + +def generate_masks_with_special_tokens_and_transfer_map(tokenized, special_tokens_list, tokenizer): + """Generate attention mask between each pair of special tokens + Args: + input_ids (torch.Tensor): input ids. Shape: [bs, num_token] + special_tokens_mask (list): special tokens mask. + Returns: + torch.Tensor: attention mask between each special tokens. + """ + input_ids = tokenized["input_ids"] + bs, num_token = input_ids.shape + # special_tokens_mask: bs, num_token. 1 for special tokens. 0 for normal tokens + special_tokens_mask = torch.zeros((bs, num_token), device=input_ids.device).bool() + for special_token in special_tokens_list: + special_tokens_mask |= input_ids == special_token + + # idxs: each row is a list of indices of special tokens + idxs = torch.nonzero(special_tokens_mask) + + # generate attention mask and positional ids + attention_mask = ( + torch.eye(num_token, device=input_ids.device).bool().unsqueeze(0).repeat(bs, 1, 1) + ) + position_ids = torch.zeros((bs, num_token), device=input_ids.device) + cate_to_token_mask_list = [[] for _ in range(bs)] + previous_col = 0 + for i in range(idxs.shape[0]): + row, col = idxs[i] + if (col == 0) or (col == num_token - 1): + attention_mask[row, col, col] = True + position_ids[row, col] = 0 + else: + attention_mask[row, previous_col + 1 : col + 1, previous_col + 1 : col + 1] = True + position_ids[row, previous_col + 1 : col + 1] = torch.arange( + 0, col - previous_col, device=input_ids.device + ) + c2t_maski = torch.zeros((num_token), device=input_ids.device).bool() + c2t_maski[previous_col + 1 : col] = True + cate_to_token_mask_list[row].append(c2t_maski) + previous_col = col + + cate_to_token_mask_list = [ + torch.stack(cate_to_token_mask_listi, dim=0) + for cate_to_token_mask_listi in cate_to_token_mask_list + ] + + # # padding mask + # padding_mask = tokenized['attention_mask'] + # attention_mask = attention_mask & padding_mask.unsqueeze(1).bool() & padding_mask.unsqueeze(2).bool() + + return attention_mask, position_ids.to(torch.long), cate_to_token_mask_list diff --git a/models/GroundingDINO/csrc/MsDeformAttn/ms_deform_attn.h b/models/GroundingDINO/csrc/MsDeformAttn/ms_deform_attn.h new file mode 100644 index 0000000000000000000000000000000000000000..c7408eba007b424194618baa63726657e36875e3 --- /dev/null +++ b/models/GroundingDINO/csrc/MsDeformAttn/ms_deform_attn.h @@ -0,0 +1,64 @@ +/*! +************************************************************************************************** +* Deformable DETR +* Copyright (c) 2020 SenseTime. All Rights Reserved. +* Licensed under the Apache License, Version 2.0 [see LICENSE for details] +************************************************************************************************** +* Modified from https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 +************************************************************************************************** +*/ + +#pragma once + +#include "ms_deform_attn_cpu.h" + +#ifdef WITH_CUDA +#include "ms_deform_attn_cuda.h" +#endif + +namespace groundingdino { + +at::Tensor +ms_deform_attn_forward( + const at::Tensor &value, + const at::Tensor &spatial_shapes, + const at::Tensor &level_start_index, + const at::Tensor &sampling_loc, + const at::Tensor &attn_weight, + const int im2col_step) +{ + if (value.type().is_cuda()) + { +#ifdef WITH_CUDA + return ms_deform_attn_cuda_forward( + value, spatial_shapes, level_start_index, sampling_loc, attn_weight, im2col_step); +#else + AT_ERROR("Not compiled with GPU support"); +#endif + } + AT_ERROR("Not implemented on the CPU"); +} + +std::vector +ms_deform_attn_backward( + const at::Tensor &value, + const at::Tensor &spatial_shapes, + const at::Tensor &level_start_index, + const at::Tensor &sampling_loc, + const at::Tensor &attn_weight, + const at::Tensor &grad_output, + const int im2col_step) +{ + if (value.type().is_cuda()) + { +#ifdef WITH_CUDA + return ms_deform_attn_cuda_backward( + value, spatial_shapes, level_start_index, sampling_loc, attn_weight, grad_output, im2col_step); +#else + AT_ERROR("Not compiled with GPU support"); +#endif + } + AT_ERROR("Not implemented on the CPU"); +} + +} // namespace groundingdino \ No newline at end of file diff --git a/models/GroundingDINO/csrc/MsDeformAttn/ms_deform_attn_cpu.cpp b/models/GroundingDINO/csrc/MsDeformAttn/ms_deform_attn_cpu.cpp new file mode 100644 index 0000000000000000000000000000000000000000..551243fdadfd1682b5dc6628623b67a79b3f6c74 --- /dev/null +++ b/models/GroundingDINO/csrc/MsDeformAttn/ms_deform_attn_cpu.cpp @@ -0,0 +1,43 @@ +/*! +************************************************************************************************** +* Deformable DETR +* Copyright (c) 2020 SenseTime. All Rights Reserved. +* Licensed under the Apache License, Version 2.0 [see LICENSE for details] +************************************************************************************************** +* Modified from https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 +************************************************************************************************** +*/ + +#include + +#include +#include + +namespace groundingdino { + +at::Tensor +ms_deform_attn_cpu_forward( + const at::Tensor &value, + const at::Tensor &spatial_shapes, + const at::Tensor &level_start_index, + const at::Tensor &sampling_loc, + const at::Tensor &attn_weight, + const int im2col_step) +{ + AT_ERROR("Not implement on cpu"); +} + +std::vector +ms_deform_attn_cpu_backward( + const at::Tensor &value, + const at::Tensor &spatial_shapes, + const at::Tensor &level_start_index, + const at::Tensor &sampling_loc, + const at::Tensor &attn_weight, + const at::Tensor &grad_output, + const int im2col_step) +{ + AT_ERROR("Not implement on cpu"); +} + +} // namespace groundingdino diff --git a/models/GroundingDINO/csrc/MsDeformAttn/ms_deform_attn_cpu.h b/models/GroundingDINO/csrc/MsDeformAttn/ms_deform_attn_cpu.h new file mode 100644 index 0000000000000000000000000000000000000000..b2b88e8c46f19b6db0933163e57ccdb51180f517 --- /dev/null +++ b/models/GroundingDINO/csrc/MsDeformAttn/ms_deform_attn_cpu.h @@ -0,0 +1,35 @@ +/*! +************************************************************************************************** +* Deformable DETR +* Copyright (c) 2020 SenseTime. All Rights Reserved. +* Licensed under the Apache License, Version 2.0 [see LICENSE for details] +************************************************************************************************** +* Modified from https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 +************************************************************************************************** +*/ + +#pragma once +#include + +namespace groundingdino { + +at::Tensor +ms_deform_attn_cpu_forward( + const at::Tensor &value, + const at::Tensor &spatial_shapes, + const at::Tensor &level_start_index, + const at::Tensor &sampling_loc, + const at::Tensor &attn_weight, + const int im2col_step); + +std::vector +ms_deform_attn_cpu_backward( + const at::Tensor &value, + const at::Tensor &spatial_shapes, + const at::Tensor &level_start_index, + const at::Tensor &sampling_loc, + const at::Tensor &attn_weight, + const at::Tensor &grad_output, + const int im2col_step); + +} // namespace groundingdino diff --git a/models/GroundingDINO/csrc/MsDeformAttn/ms_deform_attn_cuda.cu b/models/GroundingDINO/csrc/MsDeformAttn/ms_deform_attn_cuda.cu new file mode 100644 index 0000000000000000000000000000000000000000..d04fae8a9a45c11e4e74f3035e94762796da4096 --- /dev/null +++ b/models/GroundingDINO/csrc/MsDeformAttn/ms_deform_attn_cuda.cu @@ -0,0 +1,156 @@ +/*! +************************************************************************************************** +* Deformable DETR +* Copyright (c) 2020 SenseTime. All Rights Reserved. +* Licensed under the Apache License, Version 2.0 [see LICENSE for details] +************************************************************************************************** +* Modified from https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 +************************************************************************************************** +*/ + +#include +#include "ms_deform_im2col_cuda.cuh" + +#include +#include +#include +#include + +namespace groundingdino { + +at::Tensor ms_deform_attn_cuda_forward( + const at::Tensor &value, + const at::Tensor &spatial_shapes, + const at::Tensor &level_start_index, + const at::Tensor &sampling_loc, + const at::Tensor &attn_weight, + const int im2col_step) +{ + AT_ASSERTM(value.is_contiguous(), "value tensor has to be contiguous"); + AT_ASSERTM(spatial_shapes.is_contiguous(), "spatial_shapes tensor has to be contiguous"); + AT_ASSERTM(level_start_index.is_contiguous(), "level_start_index tensor has to be contiguous"); + AT_ASSERTM(sampling_loc.is_contiguous(), "sampling_loc tensor has to be contiguous"); + AT_ASSERTM(attn_weight.is_contiguous(), "attn_weight tensor has to be contiguous"); + + AT_ASSERTM(value.type().is_cuda(), "value must be a CUDA tensor"); + AT_ASSERTM(spatial_shapes.type().is_cuda(), "spatial_shapes must be a CUDA tensor"); + AT_ASSERTM(level_start_index.type().is_cuda(), "level_start_index must be a CUDA tensor"); + AT_ASSERTM(sampling_loc.type().is_cuda(), "sampling_loc must be a CUDA tensor"); + AT_ASSERTM(attn_weight.type().is_cuda(), "attn_weight must be a CUDA tensor"); + + const int batch = value.size(0); + const int spatial_size = value.size(1); + const int num_heads = value.size(2); + const int channels = value.size(3); + + const int num_levels = spatial_shapes.size(0); + + const int num_query = sampling_loc.size(1); + const int num_point = sampling_loc.size(4); + + const int im2col_step_ = std::min(batch, im2col_step); + + AT_ASSERTM(batch % im2col_step_ == 0, "batch(%d) must divide im2col_step(%d)", batch, im2col_step_); + + auto output = at::zeros({batch, num_query, num_heads, channels}, value.options()); + + const int batch_n = im2col_step_; + auto output_n = output.view({batch/im2col_step_, batch_n, num_query, num_heads, channels}); + auto per_value_size = spatial_size * num_heads * channels; + auto per_sample_loc_size = num_query * num_heads * num_levels * num_point * 2; + auto per_attn_weight_size = num_query * num_heads * num_levels * num_point; + for (int n = 0; n < batch/im2col_step_; ++n) + { + auto columns = output_n.select(0, n); + AT_DISPATCH_FLOATING_TYPES(value.type(), "ms_deform_attn_forward_cuda", ([&] { + ms_deformable_im2col_cuda(at::cuda::getCurrentCUDAStream(), + value.data() + n * im2col_step_ * per_value_size, + spatial_shapes.data(), + level_start_index.data(), + sampling_loc.data() + n * im2col_step_ * per_sample_loc_size, + attn_weight.data() + n * im2col_step_ * per_attn_weight_size, + batch_n, spatial_size, num_heads, channels, num_levels, num_query, num_point, + columns.data()); + + })); + } + + output = output.view({batch, num_query, num_heads*channels}); + + return output; +} + + +std::vector ms_deform_attn_cuda_backward( + const at::Tensor &value, + const at::Tensor &spatial_shapes, + const at::Tensor &level_start_index, + const at::Tensor &sampling_loc, + const at::Tensor &attn_weight, + const at::Tensor &grad_output, + const int im2col_step) +{ + + AT_ASSERTM(value.is_contiguous(), "value tensor has to be contiguous"); + AT_ASSERTM(spatial_shapes.is_contiguous(), "spatial_shapes tensor has to be contiguous"); + AT_ASSERTM(level_start_index.is_contiguous(), "level_start_index tensor has to be contiguous"); + AT_ASSERTM(sampling_loc.is_contiguous(), "sampling_loc tensor has to be contiguous"); + AT_ASSERTM(attn_weight.is_contiguous(), "attn_weight tensor has to be contiguous"); + AT_ASSERTM(grad_output.is_contiguous(), "grad_output tensor has to be contiguous"); + + AT_ASSERTM(value.type().is_cuda(), "value must be a CUDA tensor"); + AT_ASSERTM(spatial_shapes.type().is_cuda(), "spatial_shapes must be a CUDA tensor"); + AT_ASSERTM(level_start_index.type().is_cuda(), "level_start_index must be a CUDA tensor"); + AT_ASSERTM(sampling_loc.type().is_cuda(), "sampling_loc must be a CUDA tensor"); + AT_ASSERTM(attn_weight.type().is_cuda(), "attn_weight must be a CUDA tensor"); + AT_ASSERTM(grad_output.type().is_cuda(), "grad_output must be a CUDA tensor"); + + const int batch = value.size(0); + const int spatial_size = value.size(1); + const int num_heads = value.size(2); + const int channels = value.size(3); + + const int num_levels = spatial_shapes.size(0); + + const int num_query = sampling_loc.size(1); + const int num_point = sampling_loc.size(4); + + const int im2col_step_ = std::min(batch, im2col_step); + + AT_ASSERTM(batch % im2col_step_ == 0, "batch(%d) must divide im2col_step(%d)", batch, im2col_step_); + + auto grad_value = at::zeros_like(value); + auto grad_sampling_loc = at::zeros_like(sampling_loc); + auto grad_attn_weight = at::zeros_like(attn_weight); + + const int batch_n = im2col_step_; + auto per_value_size = spatial_size * num_heads * channels; + auto per_sample_loc_size = num_query * num_heads * num_levels * num_point * 2; + auto per_attn_weight_size = num_query * num_heads * num_levels * num_point; + auto grad_output_n = grad_output.view({batch/im2col_step_, batch_n, num_query, num_heads, channels}); + + for (int n = 0; n < batch/im2col_step_; ++n) + { + auto grad_output_g = grad_output_n.select(0, n); + AT_DISPATCH_FLOATING_TYPES(value.type(), "ms_deform_attn_backward_cuda", ([&] { + ms_deformable_col2im_cuda(at::cuda::getCurrentCUDAStream(), + grad_output_g.data(), + value.data() + n * im2col_step_ * per_value_size, + spatial_shapes.data(), + level_start_index.data(), + sampling_loc.data() + n * im2col_step_ * per_sample_loc_size, + attn_weight.data() + n * im2col_step_ * per_attn_weight_size, + batch_n, spatial_size, num_heads, channels, num_levels, num_query, num_point, + grad_value.data() + n * im2col_step_ * per_value_size, + grad_sampling_loc.data() + n * im2col_step_ * per_sample_loc_size, + grad_attn_weight.data() + n * im2col_step_ * per_attn_weight_size); + + })); + } + + return { + grad_value, grad_sampling_loc, grad_attn_weight + }; +} + +} // namespace groundingdino \ No newline at end of file diff --git a/models/GroundingDINO/csrc/MsDeformAttn/ms_deform_attn_cuda.h b/models/GroundingDINO/csrc/MsDeformAttn/ms_deform_attn_cuda.h new file mode 100644 index 0000000000000000000000000000000000000000..ad1311a78f61303616504eb991aaa9c4a93d9948 --- /dev/null +++ b/models/GroundingDINO/csrc/MsDeformAttn/ms_deform_attn_cuda.h @@ -0,0 +1,33 @@ +/*! +************************************************************************************************** +* Deformable DETR +* Copyright (c) 2020 SenseTime. All Rights Reserved. +* Licensed under the Apache License, Version 2.0 [see LICENSE for details] +************************************************************************************************** +* Modified from https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 +************************************************************************************************** +*/ + +#pragma once +#include + +namespace groundingdino { + +at::Tensor ms_deform_attn_cuda_forward( + const at::Tensor &value, + const at::Tensor &spatial_shapes, + const at::Tensor &level_start_index, + const at::Tensor &sampling_loc, + const at::Tensor &attn_weight, + const int im2col_step); + +std::vector ms_deform_attn_cuda_backward( + const at::Tensor &value, + const at::Tensor &spatial_shapes, + const at::Tensor &level_start_index, + const at::Tensor &sampling_loc, + const at::Tensor &attn_weight, + const at::Tensor &grad_output, + const int im2col_step); + +} // namespace groundingdino \ No newline at end of file diff --git a/models/GroundingDINO/csrc/MsDeformAttn/ms_deform_im2col_cuda.cuh b/models/GroundingDINO/csrc/MsDeformAttn/ms_deform_im2col_cuda.cuh new file mode 100644 index 0000000000000000000000000000000000000000..6bc2acb7aea0eab2e9e91e769a16861e1652c284 --- /dev/null +++ b/models/GroundingDINO/csrc/MsDeformAttn/ms_deform_im2col_cuda.cuh @@ -0,0 +1,1327 @@ +/*! +************************************************************************** +* Deformable DETR +* Copyright (c) 2020 SenseTime. All Rights Reserved. +* Licensed under the Apache License, Version 2.0 [see LICENSE for details] +************************************************************************** +* Modified from DCN (https://github.com/msracver/Deformable-ConvNets) +* Copyright (c) 2018 Microsoft +************************************************************************** +*/ + +#include +#include +#include + +#include +#include + +#include + +#define CUDA_KERNEL_LOOP(i, n) \ + for (int i = blockIdx.x * blockDim.x + threadIdx.x; \ + i < (n); \ + i += blockDim.x * gridDim.x) + +const int CUDA_NUM_THREADS = 1024; +inline int GET_BLOCKS(const int N, const int num_threads) +{ + return (N + num_threads - 1) / num_threads; +} + + +template +__device__ scalar_t ms_deform_attn_im2col_bilinear(const scalar_t* &bottom_data, + const int &height, const int &width, const int &nheads, const int &channels, + const scalar_t &h, const scalar_t &w, const int &m, const int &c) +{ + const int h_low = floor(h); + const int w_low = floor(w); + const int h_high = h_low + 1; + const int w_high = w_low + 1; + + const scalar_t lh = h - h_low; + const scalar_t lw = w - w_low; + const scalar_t hh = 1 - lh, hw = 1 - lw; + + const int w_stride = nheads * channels; + const int h_stride = width * w_stride; + const int h_low_ptr_offset = h_low * h_stride; + const int h_high_ptr_offset = h_low_ptr_offset + h_stride; + const int w_low_ptr_offset = w_low * w_stride; + const int w_high_ptr_offset = w_low_ptr_offset + w_stride; + const int base_ptr = m * channels + c; + + scalar_t v1 = 0; + if (h_low >= 0 && w_low >= 0) + { + const int ptr1 = h_low_ptr_offset + w_low_ptr_offset + base_ptr; + v1 = bottom_data[ptr1]; + } + scalar_t v2 = 0; + if (h_low >= 0 && w_high <= width - 1) + { + const int ptr2 = h_low_ptr_offset + w_high_ptr_offset + base_ptr; + v2 = bottom_data[ptr2]; + } + scalar_t v3 = 0; + if (h_high <= height - 1 && w_low >= 0) + { + const int ptr3 = h_high_ptr_offset + w_low_ptr_offset + base_ptr; + v3 = bottom_data[ptr3]; + } + scalar_t v4 = 0; + if (h_high <= height - 1 && w_high <= width - 1) + { + const int ptr4 = h_high_ptr_offset + w_high_ptr_offset + base_ptr; + v4 = bottom_data[ptr4]; + } + + const scalar_t w1 = hh * hw, w2 = hh * lw, w3 = lh * hw, w4 = lh * lw; + + const scalar_t val = (w1 * v1 + w2 * v2 + w3 * v3 + w4 * v4); + return val; +} + + +template +__device__ void ms_deform_attn_col2im_bilinear(const scalar_t* &bottom_data, + const int &height, const int &width, const int &nheads, const int &channels, + const scalar_t &h, const scalar_t &w, const int &m, const int &c, + const scalar_t &top_grad, + const scalar_t &attn_weight, + scalar_t* &grad_value, + scalar_t* grad_sampling_loc, + scalar_t* grad_attn_weight) +{ + const int h_low = floor(h); + const int w_low = floor(w); + const int h_high = h_low + 1; + const int w_high = w_low + 1; + + const scalar_t lh = h - h_low; + const scalar_t lw = w - w_low; + const scalar_t hh = 1 - lh, hw = 1 - lw; + + const int w_stride = nheads * channels; + const int h_stride = width * w_stride; + const int h_low_ptr_offset = h_low * h_stride; + const int h_high_ptr_offset = h_low_ptr_offset + h_stride; + const int w_low_ptr_offset = w_low * w_stride; + const int w_high_ptr_offset = w_low_ptr_offset + w_stride; + const int base_ptr = m * channels + c; + + const scalar_t w1 = hh * hw, w2 = hh * lw, w3 = lh * hw, w4 = lh * lw; + const scalar_t top_grad_value = top_grad * attn_weight; + scalar_t grad_h_weight = 0, grad_w_weight = 0; + + scalar_t v1 = 0; + if (h_low >= 0 && w_low >= 0) + { + const int ptr1 = h_low_ptr_offset + w_low_ptr_offset + base_ptr; + v1 = bottom_data[ptr1]; + grad_h_weight -= hw * v1; + grad_w_weight -= hh * v1; + atomicAdd(grad_value+ptr1, w1*top_grad_value); + } + scalar_t v2 = 0; + if (h_low >= 0 && w_high <= width - 1) + { + const int ptr2 = h_low_ptr_offset + w_high_ptr_offset + base_ptr; + v2 = bottom_data[ptr2]; + grad_h_weight -= lw * v2; + grad_w_weight += hh * v2; + atomicAdd(grad_value+ptr2, w2*top_grad_value); + } + scalar_t v3 = 0; + if (h_high <= height - 1 && w_low >= 0) + { + const int ptr3 = h_high_ptr_offset + w_low_ptr_offset + base_ptr; + v3 = bottom_data[ptr3]; + grad_h_weight += hw * v3; + grad_w_weight -= lh * v3; + atomicAdd(grad_value+ptr3, w3*top_grad_value); + } + scalar_t v4 = 0; + if (h_high <= height - 1 && w_high <= width - 1) + { + const int ptr4 = h_high_ptr_offset + w_high_ptr_offset + base_ptr; + v4 = bottom_data[ptr4]; + grad_h_weight += lw * v4; + grad_w_weight += lh * v4; + atomicAdd(grad_value+ptr4, w4*top_grad_value); + } + + const scalar_t val = (w1 * v1 + w2 * v2 + w3 * v3 + w4 * v4); + *grad_attn_weight = top_grad * val; + *grad_sampling_loc = width * grad_w_weight * top_grad_value; + *(grad_sampling_loc + 1) = height * grad_h_weight * top_grad_value; +} + + +template +__device__ void ms_deform_attn_col2im_bilinear_gm(const scalar_t* &bottom_data, + const int &height, const int &width, const int &nheads, const int &channels, + const scalar_t &h, const scalar_t &w, const int &m, const int &c, + const scalar_t &top_grad, + const scalar_t &attn_weight, + scalar_t* &grad_value, + scalar_t* grad_sampling_loc, + scalar_t* grad_attn_weight) +{ + const int h_low = floor(h); + const int w_low = floor(w); + const int h_high = h_low + 1; + const int w_high = w_low + 1; + + const scalar_t lh = h - h_low; + const scalar_t lw = w - w_low; + const scalar_t hh = 1 - lh, hw = 1 - lw; + + const int w_stride = nheads * channels; + const int h_stride = width * w_stride; + const int h_low_ptr_offset = h_low * h_stride; + const int h_high_ptr_offset = h_low_ptr_offset + h_stride; + const int w_low_ptr_offset = w_low * w_stride; + const int w_high_ptr_offset = w_low_ptr_offset + w_stride; + const int base_ptr = m * channels + c; + + const scalar_t w1 = hh * hw, w2 = hh * lw, w3 = lh * hw, w4 = lh * lw; + const scalar_t top_grad_value = top_grad * attn_weight; + scalar_t grad_h_weight = 0, grad_w_weight = 0; + + scalar_t v1 = 0; + if (h_low >= 0 && w_low >= 0) + { + const int ptr1 = h_low_ptr_offset + w_low_ptr_offset + base_ptr; + v1 = bottom_data[ptr1]; + grad_h_weight -= hw * v1; + grad_w_weight -= hh * v1; + atomicAdd(grad_value+ptr1, w1*top_grad_value); + } + scalar_t v2 = 0; + if (h_low >= 0 && w_high <= width - 1) + { + const int ptr2 = h_low_ptr_offset + w_high_ptr_offset + base_ptr; + v2 = bottom_data[ptr2]; + grad_h_weight -= lw * v2; + grad_w_weight += hh * v2; + atomicAdd(grad_value+ptr2, w2*top_grad_value); + } + scalar_t v3 = 0; + if (h_high <= height - 1 && w_low >= 0) + { + const int ptr3 = h_high_ptr_offset + w_low_ptr_offset + base_ptr; + v3 = bottom_data[ptr3]; + grad_h_weight += hw * v3; + grad_w_weight -= lh * v3; + atomicAdd(grad_value+ptr3, w3*top_grad_value); + } + scalar_t v4 = 0; + if (h_high <= height - 1 && w_high <= width - 1) + { + const int ptr4 = h_high_ptr_offset + w_high_ptr_offset + base_ptr; + v4 = bottom_data[ptr4]; + grad_h_weight += lw * v4; + grad_w_weight += lh * v4; + atomicAdd(grad_value+ptr4, w4*top_grad_value); + } + + const scalar_t val = (w1 * v1 + w2 * v2 + w3 * v3 + w4 * v4); + atomicAdd(grad_attn_weight, top_grad * val); + atomicAdd(grad_sampling_loc, width * grad_w_weight * top_grad_value); + atomicAdd(grad_sampling_loc + 1, height * grad_h_weight * top_grad_value); +} + + +template +__global__ void ms_deformable_im2col_gpu_kernel(const int n, + const scalar_t *data_value, + const int64_t *data_spatial_shapes, + const int64_t *data_level_start_index, + const scalar_t *data_sampling_loc, + const scalar_t *data_attn_weight, + const int batch_size, + const int spatial_size, + const int num_heads, + const int channels, + const int num_levels, + const int num_query, + const int num_point, + scalar_t *data_col) +{ + CUDA_KERNEL_LOOP(index, n) + { + int _temp = index; + const int c_col = _temp % channels; + _temp /= channels; + const int sampling_index = _temp; + const int m_col = _temp % num_heads; + _temp /= num_heads; + const int q_col = _temp % num_query; + _temp /= num_query; + const int b_col = _temp; + + scalar_t *data_col_ptr = data_col + index; + int data_weight_ptr = sampling_index * num_levels * num_point; + int data_loc_w_ptr = data_weight_ptr << 1; + const int qid_stride = num_heads * channels; + const int data_value_ptr_init_offset = b_col * spatial_size * qid_stride; + scalar_t col = 0; + + for (int l_col=0; l_col < num_levels; ++l_col) + { + const int level_start_id = data_level_start_index[l_col]; + const int spatial_h_ptr = l_col << 1; + const int spatial_h = data_spatial_shapes[spatial_h_ptr]; + const int spatial_w = data_spatial_shapes[spatial_h_ptr + 1]; + const scalar_t *data_value_ptr = data_value + (data_value_ptr_init_offset + level_start_id * qid_stride); + for (int p_col=0; p_col < num_point; ++p_col) + { + const scalar_t loc_w = data_sampling_loc[data_loc_w_ptr]; + const scalar_t loc_h = data_sampling_loc[data_loc_w_ptr + 1]; + const scalar_t weight = data_attn_weight[data_weight_ptr]; + + const scalar_t h_im = loc_h * spatial_h - 0.5; + const scalar_t w_im = loc_w * spatial_w - 0.5; + + if (h_im > -1 && w_im > -1 && h_im < spatial_h && w_im < spatial_w) + { + col += ms_deform_attn_im2col_bilinear(data_value_ptr, spatial_h, spatial_w, num_heads, channels, h_im, w_im, m_col, c_col) * weight; + } + + data_weight_ptr += 1; + data_loc_w_ptr += 2; + } + } + *data_col_ptr = col; + } +} + +template +__global__ void ms_deformable_col2im_gpu_kernel_shm_blocksize_aware_reduce_v1(const int n, + const scalar_t *grad_col, + const scalar_t *data_value, + const int64_t *data_spatial_shapes, + const int64_t *data_level_start_index, + const scalar_t *data_sampling_loc, + const scalar_t *data_attn_weight, + const int batch_size, + const int spatial_size, + const int num_heads, + const int channels, + const int num_levels, + const int num_query, + const int num_point, + scalar_t *grad_value, + scalar_t *grad_sampling_loc, + scalar_t *grad_attn_weight) +{ + CUDA_KERNEL_LOOP(index, n) + { + __shared__ scalar_t cache_grad_sampling_loc[blockSize * 2]; + __shared__ scalar_t cache_grad_attn_weight[blockSize]; + unsigned int tid = threadIdx.x; + int _temp = index; + const int c_col = _temp % channels; + _temp /= channels; + const int sampling_index = _temp; + const int m_col = _temp % num_heads; + _temp /= num_heads; + const int q_col = _temp % num_query; + _temp /= num_query; + const int b_col = _temp; + + const scalar_t top_grad = grad_col[index]; + + int data_weight_ptr = sampling_index * num_levels * num_point; + int data_loc_w_ptr = data_weight_ptr << 1; + const int grad_sampling_ptr = data_weight_ptr; + grad_sampling_loc += grad_sampling_ptr << 1; + grad_attn_weight += grad_sampling_ptr; + const int grad_weight_stride = 1; + const int grad_loc_stride = 2; + const int qid_stride = num_heads * channels; + const int data_value_ptr_init_offset = b_col * spatial_size * qid_stride; + + for (int l_col=0; l_col < num_levels; ++l_col) + { + const int level_start_id = data_level_start_index[l_col]; + const int spatial_h_ptr = l_col << 1; + const int spatial_h = data_spatial_shapes[spatial_h_ptr]; + const int spatial_w = data_spatial_shapes[spatial_h_ptr + 1]; + const int value_ptr_offset = data_value_ptr_init_offset + level_start_id * qid_stride; + const scalar_t *data_value_ptr = data_value + value_ptr_offset; + scalar_t *grad_value_ptr = grad_value + value_ptr_offset; + + for (int p_col=0; p_col < num_point; ++p_col) + { + const scalar_t loc_w = data_sampling_loc[data_loc_w_ptr]; + const scalar_t loc_h = data_sampling_loc[data_loc_w_ptr + 1]; + const scalar_t weight = data_attn_weight[data_weight_ptr]; + + const scalar_t h_im = loc_h * spatial_h - 0.5; + const scalar_t w_im = loc_w * spatial_w - 0.5; + *(cache_grad_sampling_loc+(threadIdx.x << 1)) = 0; + *(cache_grad_sampling_loc+((threadIdx.x << 1) + 1)) = 0; + *(cache_grad_attn_weight+threadIdx.x)=0; + if (h_im > -1 && w_im > -1 && h_im < spatial_h && w_im < spatial_w) + { + ms_deform_attn_col2im_bilinear( + data_value_ptr, spatial_h, spatial_w, num_heads, channels, h_im, w_im, m_col, c_col, + top_grad, weight, grad_value_ptr, + cache_grad_sampling_loc+(threadIdx.x << 1), cache_grad_attn_weight+threadIdx.x); + } + + __syncthreads(); + if (tid == 0) + { + scalar_t _grad_w=cache_grad_sampling_loc[0], _grad_h=cache_grad_sampling_loc[1], _grad_a=cache_grad_attn_weight[0]; + int sid=2; + for (unsigned int tid = 1; tid < blockSize; ++tid) + { + _grad_w += cache_grad_sampling_loc[sid]; + _grad_h += cache_grad_sampling_loc[sid + 1]; + _grad_a += cache_grad_attn_weight[tid]; + sid += 2; + } + + + *grad_sampling_loc = _grad_w; + *(grad_sampling_loc + 1) = _grad_h; + *grad_attn_weight = _grad_a; + } + __syncthreads(); + + data_weight_ptr += 1; + data_loc_w_ptr += 2; + grad_attn_weight += grad_weight_stride; + grad_sampling_loc += grad_loc_stride; + } + } + } +} + + +template +__global__ void ms_deformable_col2im_gpu_kernel_shm_blocksize_aware_reduce_v2(const int n, + const scalar_t *grad_col, + const scalar_t *data_value, + const int64_t *data_spatial_shapes, + const int64_t *data_level_start_index, + const scalar_t *data_sampling_loc, + const scalar_t *data_attn_weight, + const int batch_size, + const int spatial_size, + const int num_heads, + const int channels, + const int num_levels, + const int num_query, + const int num_point, + scalar_t *grad_value, + scalar_t *grad_sampling_loc, + scalar_t *grad_attn_weight) +{ + CUDA_KERNEL_LOOP(index, n) + { + __shared__ scalar_t cache_grad_sampling_loc[blockSize * 2]; + __shared__ scalar_t cache_grad_attn_weight[blockSize]; + unsigned int tid = threadIdx.x; + int _temp = index; + const int c_col = _temp % channels; + _temp /= channels; + const int sampling_index = _temp; + const int m_col = _temp % num_heads; + _temp /= num_heads; + const int q_col = _temp % num_query; + _temp /= num_query; + const int b_col = _temp; + + const scalar_t top_grad = grad_col[index]; + + int data_weight_ptr = sampling_index * num_levels * num_point; + int data_loc_w_ptr = data_weight_ptr << 1; + const int grad_sampling_ptr = data_weight_ptr; + grad_sampling_loc += grad_sampling_ptr << 1; + grad_attn_weight += grad_sampling_ptr; + const int grad_weight_stride = 1; + const int grad_loc_stride = 2; + const int qid_stride = num_heads * channels; + const int data_value_ptr_init_offset = b_col * spatial_size * qid_stride; + + for (int l_col=0; l_col < num_levels; ++l_col) + { + const int level_start_id = data_level_start_index[l_col]; + const int spatial_h_ptr = l_col << 1; + const int spatial_h = data_spatial_shapes[spatial_h_ptr]; + const int spatial_w = data_spatial_shapes[spatial_h_ptr + 1]; + const int value_ptr_offset = data_value_ptr_init_offset + level_start_id * qid_stride; + const scalar_t *data_value_ptr = data_value + value_ptr_offset; + scalar_t *grad_value_ptr = grad_value + value_ptr_offset; + + for (int p_col=0; p_col < num_point; ++p_col) + { + const scalar_t loc_w = data_sampling_loc[data_loc_w_ptr]; + const scalar_t loc_h = data_sampling_loc[data_loc_w_ptr + 1]; + const scalar_t weight = data_attn_weight[data_weight_ptr]; + + const scalar_t h_im = loc_h * spatial_h - 0.5; + const scalar_t w_im = loc_w * spatial_w - 0.5; + *(cache_grad_sampling_loc+(threadIdx.x << 1)) = 0; + *(cache_grad_sampling_loc+((threadIdx.x << 1) + 1)) = 0; + *(cache_grad_attn_weight+threadIdx.x)=0; + if (h_im > -1 && w_im > -1 && h_im < spatial_h && w_im < spatial_w) + { + ms_deform_attn_col2im_bilinear( + data_value_ptr, spatial_h, spatial_w, num_heads, channels, h_im, w_im, m_col, c_col, + top_grad, weight, grad_value_ptr, + cache_grad_sampling_loc+(threadIdx.x << 1), cache_grad_attn_weight+threadIdx.x); + } + + __syncthreads(); + + for (unsigned int s=blockSize/2; s>0; s>>=1) + { + if (tid < s) { + const unsigned int xid1 = tid << 1; + const unsigned int xid2 = (tid + s) << 1; + cache_grad_attn_weight[tid] += cache_grad_attn_weight[tid + s]; + cache_grad_sampling_loc[xid1] += cache_grad_sampling_loc[xid2]; + cache_grad_sampling_loc[xid1 + 1] += cache_grad_sampling_loc[xid2 + 1]; + } + __syncthreads(); + } + + if (tid == 0) + { + *grad_sampling_loc = cache_grad_sampling_loc[0]; + *(grad_sampling_loc + 1) = cache_grad_sampling_loc[1]; + *grad_attn_weight = cache_grad_attn_weight[0]; + } + __syncthreads(); + + data_weight_ptr += 1; + data_loc_w_ptr += 2; + grad_attn_weight += grad_weight_stride; + grad_sampling_loc += grad_loc_stride; + } + } + } +} + + +template +__global__ void ms_deformable_col2im_gpu_kernel_shm_reduce_v1(const int n, + const scalar_t *grad_col, + const scalar_t *data_value, + const int64_t *data_spatial_shapes, + const int64_t *data_level_start_index, + const scalar_t *data_sampling_loc, + const scalar_t *data_attn_weight, + const int batch_size, + const int spatial_size, + const int num_heads, + const int channels, + const int num_levels, + const int num_query, + const int num_point, + scalar_t *grad_value, + scalar_t *grad_sampling_loc, + scalar_t *grad_attn_weight) +{ + CUDA_KERNEL_LOOP(index, n) + { + extern __shared__ int _s[]; + scalar_t* cache_grad_sampling_loc = (scalar_t*)_s; + scalar_t* cache_grad_attn_weight = cache_grad_sampling_loc + 2 * blockDim.x; + unsigned int tid = threadIdx.x; + int _temp = index; + const int c_col = _temp % channels; + _temp /= channels; + const int sampling_index = _temp; + const int m_col = _temp % num_heads; + _temp /= num_heads; + const int q_col = _temp % num_query; + _temp /= num_query; + const int b_col = _temp; + + const scalar_t top_grad = grad_col[index]; + + int data_weight_ptr = sampling_index * num_levels * num_point; + int data_loc_w_ptr = data_weight_ptr << 1; + const int grad_sampling_ptr = data_weight_ptr; + grad_sampling_loc += grad_sampling_ptr << 1; + grad_attn_weight += grad_sampling_ptr; + const int grad_weight_stride = 1; + const int grad_loc_stride = 2; + const int qid_stride = num_heads * channels; + const int data_value_ptr_init_offset = b_col * spatial_size * qid_stride; + + for (int l_col=0; l_col < num_levels; ++l_col) + { + const int level_start_id = data_level_start_index[l_col]; + const int spatial_h_ptr = l_col << 1; + const int spatial_h = data_spatial_shapes[spatial_h_ptr]; + const int spatial_w = data_spatial_shapes[spatial_h_ptr + 1]; + const int value_ptr_offset = data_value_ptr_init_offset + level_start_id * qid_stride; + const scalar_t *data_value_ptr = data_value + value_ptr_offset; + scalar_t *grad_value_ptr = grad_value + value_ptr_offset; + + for (int p_col=0; p_col < num_point; ++p_col) + { + const scalar_t loc_w = data_sampling_loc[data_loc_w_ptr]; + const scalar_t loc_h = data_sampling_loc[data_loc_w_ptr + 1]; + const scalar_t weight = data_attn_weight[data_weight_ptr]; + + const scalar_t h_im = loc_h * spatial_h - 0.5; + const scalar_t w_im = loc_w * spatial_w - 0.5; + *(cache_grad_sampling_loc+(threadIdx.x << 1)) = 0; + *(cache_grad_sampling_loc+((threadIdx.x << 1) + 1)) = 0; + *(cache_grad_attn_weight+threadIdx.x)=0; + if (h_im > -1 && w_im > -1 && h_im < spatial_h && w_im < spatial_w) + { + ms_deform_attn_col2im_bilinear( + data_value_ptr, spatial_h, spatial_w, num_heads, channels, h_im, w_im, m_col, c_col, + top_grad, weight, grad_value_ptr, + cache_grad_sampling_loc+(threadIdx.x << 1), cache_grad_attn_weight+threadIdx.x); + } + + __syncthreads(); + if (tid == 0) + { + scalar_t _grad_w=cache_grad_sampling_loc[0], _grad_h=cache_grad_sampling_loc[1], _grad_a=cache_grad_attn_weight[0]; + int sid=2; + for (unsigned int tid = 1; tid < blockDim.x; ++tid) + { + _grad_w += cache_grad_sampling_loc[sid]; + _grad_h += cache_grad_sampling_loc[sid + 1]; + _grad_a += cache_grad_attn_weight[tid]; + sid += 2; + } + + + *grad_sampling_loc = _grad_w; + *(grad_sampling_loc + 1) = _grad_h; + *grad_attn_weight = _grad_a; + } + __syncthreads(); + + data_weight_ptr += 1; + data_loc_w_ptr += 2; + grad_attn_weight += grad_weight_stride; + grad_sampling_loc += grad_loc_stride; + } + } + } +} + +template +__global__ void ms_deformable_col2im_gpu_kernel_shm_reduce_v2(const int n, + const scalar_t *grad_col, + const scalar_t *data_value, + const int64_t *data_spatial_shapes, + const int64_t *data_level_start_index, + const scalar_t *data_sampling_loc, + const scalar_t *data_attn_weight, + const int batch_size, + const int spatial_size, + const int num_heads, + const int channels, + const int num_levels, + const int num_query, + const int num_point, + scalar_t *grad_value, + scalar_t *grad_sampling_loc, + scalar_t *grad_attn_weight) +{ + CUDA_KERNEL_LOOP(index, n) + { + extern __shared__ int _s[]; + scalar_t* cache_grad_sampling_loc = (scalar_t*)_s; + scalar_t* cache_grad_attn_weight = cache_grad_sampling_loc + 2 * blockDim.x; + unsigned int tid = threadIdx.x; + int _temp = index; + const int c_col = _temp % channels; + _temp /= channels; + const int sampling_index = _temp; + const int m_col = _temp % num_heads; + _temp /= num_heads; + const int q_col = _temp % num_query; + _temp /= num_query; + const int b_col = _temp; + + const scalar_t top_grad = grad_col[index]; + + int data_weight_ptr = sampling_index * num_levels * num_point; + int data_loc_w_ptr = data_weight_ptr << 1; + const int grad_sampling_ptr = data_weight_ptr; + grad_sampling_loc += grad_sampling_ptr << 1; + grad_attn_weight += grad_sampling_ptr; + const int grad_weight_stride = 1; + const int grad_loc_stride = 2; + const int qid_stride = num_heads * channels; + const int data_value_ptr_init_offset = b_col * spatial_size * qid_stride; + + for (int l_col=0; l_col < num_levels; ++l_col) + { + const int level_start_id = data_level_start_index[l_col]; + const int spatial_h_ptr = l_col << 1; + const int spatial_h = data_spatial_shapes[spatial_h_ptr]; + const int spatial_w = data_spatial_shapes[spatial_h_ptr + 1]; + const int value_ptr_offset = data_value_ptr_init_offset + level_start_id * qid_stride; + const scalar_t *data_value_ptr = data_value + value_ptr_offset; + scalar_t *grad_value_ptr = grad_value + value_ptr_offset; + + for (int p_col=0; p_col < num_point; ++p_col) + { + const scalar_t loc_w = data_sampling_loc[data_loc_w_ptr]; + const scalar_t loc_h = data_sampling_loc[data_loc_w_ptr + 1]; + const scalar_t weight = data_attn_weight[data_weight_ptr]; + + const scalar_t h_im = loc_h * spatial_h - 0.5; + const scalar_t w_im = loc_w * spatial_w - 0.5; + *(cache_grad_sampling_loc+(threadIdx.x << 1)) = 0; + *(cache_grad_sampling_loc+((threadIdx.x << 1) + 1)) = 0; + *(cache_grad_attn_weight+threadIdx.x)=0; + if (h_im > -1 && w_im > -1 && h_im < spatial_h && w_im < spatial_w) + { + ms_deform_attn_col2im_bilinear( + data_value_ptr, spatial_h, spatial_w, num_heads, channels, h_im, w_im, m_col, c_col, + top_grad, weight, grad_value_ptr, + cache_grad_sampling_loc+(threadIdx.x << 1), cache_grad_attn_weight+threadIdx.x); + } + + __syncthreads(); + + for (unsigned int s=blockDim.x/2, spre=blockDim.x; s>0; s>>=1, spre>>=1) + { + if (tid < s) { + const unsigned int xid1 = tid << 1; + const unsigned int xid2 = (tid + s) << 1; + cache_grad_attn_weight[tid] += cache_grad_attn_weight[tid + s]; + cache_grad_sampling_loc[xid1] += cache_grad_sampling_loc[xid2]; + cache_grad_sampling_loc[xid1 + 1] += cache_grad_sampling_loc[xid2 + 1]; + if (tid + (s << 1) < spre) + { + cache_grad_attn_weight[tid] += cache_grad_attn_weight[tid + (s << 1)]; + cache_grad_sampling_loc[xid1] += cache_grad_sampling_loc[xid2 + (s << 1)]; + cache_grad_sampling_loc[xid1 + 1] += cache_grad_sampling_loc[xid2 + 1 + (s << 1)]; + } + } + __syncthreads(); + } + + if (tid == 0) + { + *grad_sampling_loc = cache_grad_sampling_loc[0]; + *(grad_sampling_loc + 1) = cache_grad_sampling_loc[1]; + *grad_attn_weight = cache_grad_attn_weight[0]; + } + __syncthreads(); + + data_weight_ptr += 1; + data_loc_w_ptr += 2; + grad_attn_weight += grad_weight_stride; + grad_sampling_loc += grad_loc_stride; + } + } + } +} + +template +__global__ void ms_deformable_col2im_gpu_kernel_shm_reduce_v2_multi_blocks(const int n, + const scalar_t *grad_col, + const scalar_t *data_value, + const int64_t *data_spatial_shapes, + const int64_t *data_level_start_index, + const scalar_t *data_sampling_loc, + const scalar_t *data_attn_weight, + const int batch_size, + const int spatial_size, + const int num_heads, + const int channels, + const int num_levels, + const int num_query, + const int num_point, + scalar_t *grad_value, + scalar_t *grad_sampling_loc, + scalar_t *grad_attn_weight) +{ + CUDA_KERNEL_LOOP(index, n) + { + extern __shared__ int _s[]; + scalar_t* cache_grad_sampling_loc = (scalar_t*)_s; + scalar_t* cache_grad_attn_weight = cache_grad_sampling_loc + 2 * blockDim.x; + unsigned int tid = threadIdx.x; + int _temp = index; + const int c_col = _temp % channels; + _temp /= channels; + const int sampling_index = _temp; + const int m_col = _temp % num_heads; + _temp /= num_heads; + const int q_col = _temp % num_query; + _temp /= num_query; + const int b_col = _temp; + + const scalar_t top_grad = grad_col[index]; + + int data_weight_ptr = sampling_index * num_levels * num_point; + int data_loc_w_ptr = data_weight_ptr << 1; + const int grad_sampling_ptr = data_weight_ptr; + grad_sampling_loc += grad_sampling_ptr << 1; + grad_attn_weight += grad_sampling_ptr; + const int grad_weight_stride = 1; + const int grad_loc_stride = 2; + const int qid_stride = num_heads * channels; + const int data_value_ptr_init_offset = b_col * spatial_size * qid_stride; + + for (int l_col=0; l_col < num_levels; ++l_col) + { + const int level_start_id = data_level_start_index[l_col]; + const int spatial_h_ptr = l_col << 1; + const int spatial_h = data_spatial_shapes[spatial_h_ptr]; + const int spatial_w = data_spatial_shapes[spatial_h_ptr + 1]; + const int value_ptr_offset = data_value_ptr_init_offset + level_start_id * qid_stride; + const scalar_t *data_value_ptr = data_value + value_ptr_offset; + scalar_t *grad_value_ptr = grad_value + value_ptr_offset; + + for (int p_col=0; p_col < num_point; ++p_col) + { + const scalar_t loc_w = data_sampling_loc[data_loc_w_ptr]; + const scalar_t loc_h = data_sampling_loc[data_loc_w_ptr + 1]; + const scalar_t weight = data_attn_weight[data_weight_ptr]; + + const scalar_t h_im = loc_h * spatial_h - 0.5; + const scalar_t w_im = loc_w * spatial_w - 0.5; + *(cache_grad_sampling_loc+(threadIdx.x << 1)) = 0; + *(cache_grad_sampling_loc+((threadIdx.x << 1) + 1)) = 0; + *(cache_grad_attn_weight+threadIdx.x)=0; + if (h_im > -1 && w_im > -1 && h_im < spatial_h && w_im < spatial_w) + { + ms_deform_attn_col2im_bilinear( + data_value_ptr, spatial_h, spatial_w, num_heads, channels, h_im, w_im, m_col, c_col, + top_grad, weight, grad_value_ptr, + cache_grad_sampling_loc+(threadIdx.x << 1), cache_grad_attn_weight+threadIdx.x); + } + + __syncthreads(); + + for (unsigned int s=blockDim.x/2, spre=blockDim.x; s>0; s>>=1, spre>>=1) + { + if (tid < s) { + const unsigned int xid1 = tid << 1; + const unsigned int xid2 = (tid + s) << 1; + cache_grad_attn_weight[tid] += cache_grad_attn_weight[tid + s]; + cache_grad_sampling_loc[xid1] += cache_grad_sampling_loc[xid2]; + cache_grad_sampling_loc[xid1 + 1] += cache_grad_sampling_loc[xid2 + 1]; + if (tid + (s << 1) < spre) + { + cache_grad_attn_weight[tid] += cache_grad_attn_weight[tid + (s << 1)]; + cache_grad_sampling_loc[xid1] += cache_grad_sampling_loc[xid2 + (s << 1)]; + cache_grad_sampling_loc[xid1 + 1] += cache_grad_sampling_loc[xid2 + 1 + (s << 1)]; + } + } + __syncthreads(); + } + + if (tid == 0) + { + atomicAdd(grad_sampling_loc, cache_grad_sampling_loc[0]); + atomicAdd(grad_sampling_loc + 1, cache_grad_sampling_loc[1]); + atomicAdd(grad_attn_weight, cache_grad_attn_weight[0]); + } + __syncthreads(); + + data_weight_ptr += 1; + data_loc_w_ptr += 2; + grad_attn_weight += grad_weight_stride; + grad_sampling_loc += grad_loc_stride; + } + } + } +} + + +template +__global__ void ms_deformable_col2im_gpu_kernel_gm(const int n, + const scalar_t *grad_col, + const scalar_t *data_value, + const int64_t *data_spatial_shapes, + const int64_t *data_level_start_index, + const scalar_t *data_sampling_loc, + const scalar_t *data_attn_weight, + const int batch_size, + const int spatial_size, + const int num_heads, + const int channels, + const int num_levels, + const int num_query, + const int num_point, + scalar_t *grad_value, + scalar_t *grad_sampling_loc, + scalar_t *grad_attn_weight) +{ + CUDA_KERNEL_LOOP(index, n) + { + int _temp = index; + const int c_col = _temp % channels; + _temp /= channels; + const int sampling_index = _temp; + const int m_col = _temp % num_heads; + _temp /= num_heads; + const int q_col = _temp % num_query; + _temp /= num_query; + const int b_col = _temp; + + const scalar_t top_grad = grad_col[index]; + + int data_weight_ptr = sampling_index * num_levels * num_point; + int data_loc_w_ptr = data_weight_ptr << 1; + const int grad_sampling_ptr = data_weight_ptr; + grad_sampling_loc += grad_sampling_ptr << 1; + grad_attn_weight += grad_sampling_ptr; + const int grad_weight_stride = 1; + const int grad_loc_stride = 2; + const int qid_stride = num_heads * channels; + const int data_value_ptr_init_offset = b_col * spatial_size * qid_stride; + + for (int l_col=0; l_col < num_levels; ++l_col) + { + const int level_start_id = data_level_start_index[l_col]; + const int spatial_h_ptr = l_col << 1; + const int spatial_h = data_spatial_shapes[spatial_h_ptr]; + const int spatial_w = data_spatial_shapes[spatial_h_ptr + 1]; + const int value_ptr_offset = data_value_ptr_init_offset + level_start_id * qid_stride; + const scalar_t *data_value_ptr = data_value + value_ptr_offset; + scalar_t *grad_value_ptr = grad_value + value_ptr_offset; + + for (int p_col=0; p_col < num_point; ++p_col) + { + const scalar_t loc_w = data_sampling_loc[data_loc_w_ptr]; + const scalar_t loc_h = data_sampling_loc[data_loc_w_ptr + 1]; + const scalar_t weight = data_attn_weight[data_weight_ptr]; + + const scalar_t h_im = loc_h * spatial_h - 0.5; + const scalar_t w_im = loc_w * spatial_w - 0.5; + if (h_im > -1 && w_im > -1 && h_im < spatial_h && w_im < spatial_w) + { + ms_deform_attn_col2im_bilinear_gm( + data_value_ptr, spatial_h, spatial_w, num_heads, channels, h_im, w_im, m_col, c_col, + top_grad, weight, grad_value_ptr, + grad_sampling_loc, grad_attn_weight); + } + data_weight_ptr += 1; + data_loc_w_ptr += 2; + grad_attn_weight += grad_weight_stride; + grad_sampling_loc += grad_loc_stride; + } + } + } +} + + +template +void ms_deformable_im2col_cuda(cudaStream_t stream, + const scalar_t* data_value, + const int64_t* data_spatial_shapes, + const int64_t* data_level_start_index, + const scalar_t* data_sampling_loc, + const scalar_t* data_attn_weight, + const int batch_size, + const int spatial_size, + const int num_heads, + const int channels, + const int num_levels, + const int num_query, + const int num_point, + scalar_t* data_col) +{ + const int num_kernels = batch_size * num_query * num_heads * channels; + const int num_actual_kernels = batch_size * num_query * num_heads * channels; + const int num_threads = CUDA_NUM_THREADS; + ms_deformable_im2col_gpu_kernel + <<>>( + num_kernels, data_value, data_spatial_shapes, data_level_start_index, data_sampling_loc, data_attn_weight, + batch_size, spatial_size, num_heads, channels, num_levels, num_query, num_point, data_col); + + cudaError_t err = cudaGetLastError(); + if (err != cudaSuccess) + { + printf("error in ms_deformable_im2col_cuda: %s\n", cudaGetErrorString(err)); + } + +} + +template +void ms_deformable_col2im_cuda(cudaStream_t stream, + const scalar_t* grad_col, + const scalar_t* data_value, + const int64_t * data_spatial_shapes, + const int64_t * data_level_start_index, + const scalar_t * data_sampling_loc, + const scalar_t * data_attn_weight, + const int batch_size, + const int spatial_size, + const int num_heads, + const int channels, + const int num_levels, + const int num_query, + const int num_point, + scalar_t* grad_value, + scalar_t* grad_sampling_loc, + scalar_t* grad_attn_weight) +{ + const int num_threads = (channels > CUDA_NUM_THREADS)?CUDA_NUM_THREADS:channels; + const int num_kernels = batch_size * num_query * num_heads * channels; + const int num_actual_kernels = batch_size * num_query * num_heads * channels; + if (channels > 1024) + { + if ((channels & 1023) == 0) + { + ms_deformable_col2im_gpu_kernel_shm_reduce_v2_multi_blocks + <<>>( + num_kernels, + grad_col, + data_value, + data_spatial_shapes, + data_level_start_index, + data_sampling_loc, + data_attn_weight, + batch_size, + spatial_size, + num_heads, + channels, + num_levels, + num_query, + num_point, + grad_value, + grad_sampling_loc, + grad_attn_weight); + } + else + { + ms_deformable_col2im_gpu_kernel_gm + <<>>( + num_kernels, + grad_col, + data_value, + data_spatial_shapes, + data_level_start_index, + data_sampling_loc, + data_attn_weight, + batch_size, + spatial_size, + num_heads, + channels, + num_levels, + num_query, + num_point, + grad_value, + grad_sampling_loc, + grad_attn_weight); + } + } + else{ + switch(channels) + { + case 1: + ms_deformable_col2im_gpu_kernel_shm_blocksize_aware_reduce_v1 + <<>>( + num_kernels, + grad_col, + data_value, + data_spatial_shapes, + data_level_start_index, + data_sampling_loc, + data_attn_weight, + batch_size, + spatial_size, + num_heads, + channels, + num_levels, + num_query, + num_point, + grad_value, + grad_sampling_loc, + grad_attn_weight); + break; + case 2: + ms_deformable_col2im_gpu_kernel_shm_blocksize_aware_reduce_v1 + <<>>( + num_kernels, + grad_col, + data_value, + data_spatial_shapes, + data_level_start_index, + data_sampling_loc, + data_attn_weight, + batch_size, + spatial_size, + num_heads, + channels, + num_levels, + num_query, + num_point, + grad_value, + grad_sampling_loc, + grad_attn_weight); + break; + case 4: + ms_deformable_col2im_gpu_kernel_shm_blocksize_aware_reduce_v1 + <<>>( + num_kernels, + grad_col, + data_value, + data_spatial_shapes, + data_level_start_index, + data_sampling_loc, + data_attn_weight, + batch_size, + spatial_size, + num_heads, + channels, + num_levels, + num_query, + num_point, + grad_value, + grad_sampling_loc, + grad_attn_weight); + break; + case 8: + ms_deformable_col2im_gpu_kernel_shm_blocksize_aware_reduce_v1 + <<>>( + num_kernels, + grad_col, + data_value, + data_spatial_shapes, + data_level_start_index, + data_sampling_loc, + data_attn_weight, + batch_size, + spatial_size, + num_heads, + channels, + num_levels, + num_query, + num_point, + grad_value, + grad_sampling_loc, + grad_attn_weight); + break; + case 16: + ms_deformable_col2im_gpu_kernel_shm_blocksize_aware_reduce_v1 + <<>>( + num_kernels, + grad_col, + data_value, + data_spatial_shapes, + data_level_start_index, + data_sampling_loc, + data_attn_weight, + batch_size, + spatial_size, + num_heads, + channels, + num_levels, + num_query, + num_point, + grad_value, + grad_sampling_loc, + grad_attn_weight); + break; + case 32: + ms_deformable_col2im_gpu_kernel_shm_blocksize_aware_reduce_v1 + <<>>( + num_kernels, + grad_col, + data_value, + data_spatial_shapes, + data_level_start_index, + data_sampling_loc, + data_attn_weight, + batch_size, + spatial_size, + num_heads, + channels, + num_levels, + num_query, + num_point, + grad_value, + grad_sampling_loc, + grad_attn_weight); + break; + case 64: + ms_deformable_col2im_gpu_kernel_shm_blocksize_aware_reduce_v2 + <<>>( + num_kernels, + grad_col, + data_value, + data_spatial_shapes, + data_level_start_index, + data_sampling_loc, + data_attn_weight, + batch_size, + spatial_size, + num_heads, + channels, + num_levels, + num_query, + num_point, + grad_value, + grad_sampling_loc, + grad_attn_weight); + break; + case 128: + ms_deformable_col2im_gpu_kernel_shm_blocksize_aware_reduce_v2 + <<>>( + num_kernels, + grad_col, + data_value, + data_spatial_shapes, + data_level_start_index, + data_sampling_loc, + data_attn_weight, + batch_size, + spatial_size, + num_heads, + channels, + num_levels, + num_query, + num_point, + grad_value, + grad_sampling_loc, + grad_attn_weight); + break; + case 256: + ms_deformable_col2im_gpu_kernel_shm_blocksize_aware_reduce_v2 + <<>>( + num_kernels, + grad_col, + data_value, + data_spatial_shapes, + data_level_start_index, + data_sampling_loc, + data_attn_weight, + batch_size, + spatial_size, + num_heads, + channels, + num_levels, + num_query, + num_point, + grad_value, + grad_sampling_loc, + grad_attn_weight); + break; + case 512: + ms_deformable_col2im_gpu_kernel_shm_blocksize_aware_reduce_v2 + <<>>( + num_kernels, + grad_col, + data_value, + data_spatial_shapes, + data_level_start_index, + data_sampling_loc, + data_attn_weight, + batch_size, + spatial_size, + num_heads, + channels, + num_levels, + num_query, + num_point, + grad_value, + grad_sampling_loc, + grad_attn_weight); + break; + case 1024: + ms_deformable_col2im_gpu_kernel_shm_blocksize_aware_reduce_v2 + <<>>( + num_kernels, + grad_col, + data_value, + data_spatial_shapes, + data_level_start_index, + data_sampling_loc, + data_attn_weight, + batch_size, + spatial_size, + num_heads, + channels, + num_levels, + num_query, + num_point, + grad_value, + grad_sampling_loc, + grad_attn_weight); + break; + default: + if (channels < 64) + { + ms_deformable_col2im_gpu_kernel_shm_reduce_v1 + <<>>( + num_kernels, + grad_col, + data_value, + data_spatial_shapes, + data_level_start_index, + data_sampling_loc, + data_attn_weight, + batch_size, + spatial_size, + num_heads, + channels, + num_levels, + num_query, + num_point, + grad_value, + grad_sampling_loc, + grad_attn_weight); + } + else + { + ms_deformable_col2im_gpu_kernel_shm_reduce_v2 + <<>>( + num_kernels, + grad_col, + data_value, + data_spatial_shapes, + data_level_start_index, + data_sampling_loc, + data_attn_weight, + batch_size, + spatial_size, + num_heads, + channels, + num_levels, + num_query, + num_point, + grad_value, + grad_sampling_loc, + grad_attn_weight); + } + } + } + cudaError_t err = cudaGetLastError(); + if (err != cudaSuccess) + { + printf("error in ms_deformable_col2im_cuda: %s\n", cudaGetErrorString(err)); + } + +} \ No newline at end of file diff --git a/models/GroundingDINO/csrc/cuda_version.cu b/models/GroundingDINO/csrc/cuda_version.cu new file mode 100644 index 0000000000000000000000000000000000000000..64569e34ffb250964de27e33e7a53f3822270b9e --- /dev/null +++ b/models/GroundingDINO/csrc/cuda_version.cu @@ -0,0 +1,7 @@ +#include + +namespace groundingdino { +int get_cudart_version() { + return CUDART_VERSION; +} +} // namespace groundingdino diff --git a/models/GroundingDINO/csrc/vision.cpp b/models/GroundingDINO/csrc/vision.cpp new file mode 100644 index 0000000000000000000000000000000000000000..c1f2c50c82909bbd5492c163d634af77a3ba1781 --- /dev/null +++ b/models/GroundingDINO/csrc/vision.cpp @@ -0,0 +1,58 @@ +// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved + +#include "MsDeformAttn/ms_deform_attn.h" + +namespace groundingdino { + +#ifdef WITH_CUDA +extern int get_cudart_version(); +#endif + +std::string get_cuda_version() { +#ifdef WITH_CUDA + std::ostringstream oss; + + // copied from + // https://github.com/pytorch/pytorch/blob/master/aten/src/ATen/cuda/detail/CUDAHooks.cpp#L231 + auto printCudaStyleVersion = [&](int v) { + oss << (v / 1000) << "." << (v / 10 % 100); + if (v % 10 != 0) { + oss << "." << (v % 10); + } + }; + printCudaStyleVersion(get_cudart_version()); + return oss.str(); +#else + return std::string("not available"); +#endif +} + +// similar to +// https://github.com/pytorch/pytorch/blob/master/aten/src/ATen/Version.cpp +std::string get_compiler_version() { + std::ostringstream ss; +#if defined(__GNUC__) +#ifndef __clang__ + { ss << "GCC " << __GNUC__ << "." << __GNUC_MINOR__; } +#endif +#endif + +#if defined(__clang_major__) + { + ss << "clang " << __clang_major__ << "." << __clang_minor__ << "." + << __clang_patchlevel__; + } +#endif + +#if defined(_MSC_VER) + { ss << "MSVC " << _MSC_FULL_VER; } +#endif + return ss.str(); +} + +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { + m.def("ms_deform_attn_forward", &ms_deform_attn_forward, "ms_deform_attn_forward"); + m.def("ms_deform_attn_backward", &ms_deform_attn_backward, "ms_deform_attn_backward"); +} + +} // namespace groundingdino \ No newline at end of file diff --git a/models/GroundingDINO/fuse_modules.py b/models/GroundingDINO/fuse_modules.py new file mode 100644 index 0000000000000000000000000000000000000000..2753b3ddee43c7a9fe28d1824db5d786e7e1ad59 --- /dev/null +++ b/models/GroundingDINO/fuse_modules.py @@ -0,0 +1,297 @@ +# ------------------------------------------------------------------------ +# Grounding DINO +# url: https://github.com/IDEA-Research/GroundingDINO +# Copyright (c) 2023 IDEA. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------ + +import torch +import torch.nn as nn +import torch.nn.functional as F +from timm.models.layers import DropPath + + +class FeatureResizer(nn.Module): + """ + This class takes as input a set of embeddings of dimension C1 and outputs a set of + embedding of dimension C2, after a linear transformation, dropout and normalization (LN). + """ + + def __init__(self, input_feat_size, output_feat_size, dropout, do_ln=True): + super().__init__() + self.do_ln = do_ln + # Object feature encoding + self.fc = nn.Linear(input_feat_size, output_feat_size, bias=True) + self.layer_norm = nn.LayerNorm(output_feat_size, eps=1e-12) + self.dropout = nn.Dropout(dropout) + + def forward(self, encoder_features): + x = self.fc(encoder_features) + if self.do_ln: + x = self.layer_norm(x) + output = self.dropout(x) + return output + + +def l1norm(X, dim, eps=1e-8): + """L1-normalize columns of X""" + norm = torch.abs(X).sum(dim=dim, keepdim=True) + eps + X = torch.div(X, norm) + return X + + +def l2norm(X, dim, eps=1e-8): + """L2-normalize columns of X""" + norm = torch.pow(X, 2).sum(dim=dim, keepdim=True).sqrt() + eps + X = torch.div(X, norm) + return X + + +def func_attention(query, context, smooth=1, raw_feature_norm="softmax", eps=1e-8): + """ + query: (n_context, queryL, d) + context: (n_context, sourceL, d) + """ + batch_size_q, queryL = query.size(0), query.size(1) + batch_size, sourceL = context.size(0), context.size(1) + + # Get attention + # --> (batch, d, queryL) + queryT = torch.transpose(query, 1, 2) + + # (batch, sourceL, d)(batch, d, queryL) + # --> (batch, sourceL, queryL) + attn = torch.bmm(context, queryT) + if raw_feature_norm == "softmax": + # --> (batch*sourceL, queryL) + attn = attn.view(batch_size * sourceL, queryL) + attn = nn.Softmax()(attn) + # --> (batch, sourceL, queryL) + attn = attn.view(batch_size, sourceL, queryL) + elif raw_feature_norm == "l2norm": + attn = l2norm(attn, 2) + elif raw_feature_norm == "clipped_l2norm": + attn = nn.LeakyReLU(0.1)(attn) + attn = l2norm(attn, 2) + else: + raise ValueError("unknown first norm type:", raw_feature_norm) + # --> (batch, queryL, sourceL) + attn = torch.transpose(attn, 1, 2).contiguous() + # --> (batch*queryL, sourceL) + attn = attn.view(batch_size * queryL, sourceL) + attn = nn.Softmax()(attn * smooth) + # --> (batch, queryL, sourceL) + attn = attn.view(batch_size, queryL, sourceL) + # --> (batch, sourceL, queryL) + attnT = torch.transpose(attn, 1, 2).contiguous() + + # --> (batch, d, sourceL) + contextT = torch.transpose(context, 1, 2) + # (batch x d x sourceL)(batch x sourceL x queryL) + # --> (batch, d, queryL) + weightedContext = torch.bmm(contextT, attnT) + # --> (batch, queryL, d) + weightedContext = torch.transpose(weightedContext, 1, 2) + + return weightedContext, attnT + + +class BiMultiHeadAttention(nn.Module): + def __init__(self, v_dim, l_dim, embed_dim, num_heads, dropout=0.1, cfg=None): + super(BiMultiHeadAttention, self).__init__() + + self.embed_dim = embed_dim + self.num_heads = num_heads + self.head_dim = embed_dim // num_heads + self.v_dim = v_dim + self.l_dim = l_dim + + assert ( + self.head_dim * self.num_heads == self.embed_dim + ), f"embed_dim must be divisible by num_heads (got `embed_dim`: {self.embed_dim} and `num_heads`: {self.num_heads})." + self.scale = self.head_dim ** (-0.5) + self.dropout = dropout + + self.v_proj = nn.Linear(self.v_dim, self.embed_dim) + self.l_proj = nn.Linear(self.l_dim, self.embed_dim) + self.values_v_proj = nn.Linear(self.v_dim, self.embed_dim) + self.values_l_proj = nn.Linear(self.l_dim, self.embed_dim) + + self.out_v_proj = nn.Linear(self.embed_dim, self.v_dim) + self.out_l_proj = nn.Linear(self.embed_dim, self.l_dim) + + self.stable_softmax_2d = True + self.clamp_min_for_underflow = True + self.clamp_max_for_overflow = True + + self._reset_parameters() + + def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int): + return tensor.view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2).contiguous() + + def _reset_parameters(self): + nn.init.xavier_uniform_(self.v_proj.weight) + self.v_proj.bias.data.fill_(0) + nn.init.xavier_uniform_(self.l_proj.weight) + self.l_proj.bias.data.fill_(0) + nn.init.xavier_uniform_(self.values_v_proj.weight) + self.values_v_proj.bias.data.fill_(0) + nn.init.xavier_uniform_(self.values_l_proj.weight) + self.values_l_proj.bias.data.fill_(0) + nn.init.xavier_uniform_(self.out_v_proj.weight) + self.out_v_proj.bias.data.fill_(0) + nn.init.xavier_uniform_(self.out_l_proj.weight) + self.out_l_proj.bias.data.fill_(0) + + def forward(self, v, l, attention_mask_v=None, attention_mask_l=None): + """_summary_ + + Args: + v (_type_): bs, n_img, dim + l (_type_): bs, n_text, dim + attention_mask_v (_type_, optional): _description_. bs, n_img + attention_mask_l (_type_, optional): _description_. bs, n_text + + Returns: + _type_: _description_ + """ + # if os.environ.get('IPDB_SHILONG_DEBUG', None) == 'INFO': + # import ipdb; ipdb.set_trace() + bsz, tgt_len, _ = v.size() + + query_states = self.v_proj(v) * self.scale + key_states = self._shape(self.l_proj(l), -1, bsz) + value_v_states = self._shape(self.values_v_proj(v), -1, bsz) + value_l_states = self._shape(self.values_l_proj(l), -1, bsz) + + proj_shape = (bsz * self.num_heads, -1, self.head_dim) + query_states = self._shape(query_states, tgt_len, bsz).view(*proj_shape) + key_states = key_states.view(*proj_shape) + value_v_states = value_v_states.view(*proj_shape) + value_l_states = value_l_states.view(*proj_shape) + + src_len = key_states.size(1) + attn_weights = torch.bmm(query_states, key_states.transpose(1, 2)) # bs*nhead, nimg, ntxt + + if attn_weights.size() != (bsz * self.num_heads, tgt_len, src_len): + raise ValueError( + f"Attention weights should be of size {(bsz * self.num_heads, tgt_len, src_len)}, but is {attn_weights.size()}" + ) + + if self.stable_softmax_2d: + attn_weights = attn_weights - attn_weights.max() + + if self.clamp_min_for_underflow: + attn_weights = torch.clamp( + attn_weights, min=-50000 + ) # Do not increase -50000, data type half has quite limited range + if self.clamp_max_for_overflow: + attn_weights = torch.clamp( + attn_weights, max=50000 + ) # Do not increase 50000, data type half has quite limited range + + attn_weights_T = attn_weights.transpose(1, 2) + attn_weights_l = attn_weights_T - torch.max(attn_weights_T, dim=-1, keepdim=True)[0] + if self.clamp_min_for_underflow: + attn_weights_l = torch.clamp( + attn_weights_l, min=-50000 + ) # Do not increase -50000, data type half has quite limited range + if self.clamp_max_for_overflow: + attn_weights_l = torch.clamp( + attn_weights_l, max=50000 + ) # Do not increase 50000, data type half has quite limited range + + # mask vison for language + if attention_mask_v is not None: + attention_mask_v = ( + attention_mask_v[:, None, None, :].repeat(1, self.num_heads, 1, 1).flatten(0, 1) + ) + attn_weights_l.masked_fill_(attention_mask_v, float("-inf")) + + attn_weights_l = attn_weights_l.softmax(dim=-1) + + # mask language for vision + if attention_mask_l is not None: + attention_mask_l = ( + attention_mask_l[:, None, None, :].repeat(1, self.num_heads, 1, 1).flatten(0, 1) + ) + attn_weights.masked_fill_(attention_mask_l, float("-inf")) + attn_weights_v = attn_weights.softmax(dim=-1) + + attn_probs_v = F.dropout(attn_weights_v, p=self.dropout, training=self.training) + attn_probs_l = F.dropout(attn_weights_l, p=self.dropout, training=self.training) + + attn_output_v = torch.bmm(attn_probs_v, value_l_states) + attn_output_l = torch.bmm(attn_probs_l, value_v_states) + + if attn_output_v.size() != (bsz * self.num_heads, tgt_len, self.head_dim): + raise ValueError( + f"`attn_output_v` should be of size {(bsz, self.num_heads, tgt_len, self.head_dim)}, but is {attn_output_v.size()}" + ) + + if attn_output_l.size() != (bsz * self.num_heads, src_len, self.head_dim): + raise ValueError( + f"`attn_output_l` should be of size {(bsz, self.num_heads, src_len, self.head_dim)}, but is {attn_output_l.size()}" + ) + + attn_output_v = attn_output_v.view(bsz, self.num_heads, tgt_len, self.head_dim) + attn_output_v = attn_output_v.transpose(1, 2) + attn_output_v = attn_output_v.reshape(bsz, tgt_len, self.embed_dim) + + attn_output_l = attn_output_l.view(bsz, self.num_heads, src_len, self.head_dim) + attn_output_l = attn_output_l.transpose(1, 2) + attn_output_l = attn_output_l.reshape(bsz, src_len, self.embed_dim) + + attn_output_v = self.out_v_proj(attn_output_v) + attn_output_l = self.out_l_proj(attn_output_l) + + return attn_output_v, attn_output_l + + +# Bi-Direction MHA (text->image, image->text) +class BiAttentionBlock(nn.Module): + def __init__( + self, + v_dim, + l_dim, + embed_dim, + num_heads, + dropout=0.1, + drop_path=0.0, + init_values=1e-4, + cfg=None, + ): + """ + Inputs: + embed_dim - Dimensionality of input and attention feature vectors + hidden_dim - Dimensionality of hidden layer in feed-forward network + (usually 2-4x larger than embed_dim) + num_heads - Number of heads to use in the Multi-Head Attention block + dropout - Amount of dropout to apply in the feed-forward network + """ + super(BiAttentionBlock, self).__init__() + + # pre layer norm + self.layer_norm_v = nn.LayerNorm(v_dim) + self.layer_norm_l = nn.LayerNorm(l_dim) + self.attn = BiMultiHeadAttention( + v_dim=v_dim, l_dim=l_dim, embed_dim=embed_dim, num_heads=num_heads, dropout=dropout + ) + + # add layer scale for training stability + self.drop_path = DropPath(drop_path) if drop_path > 0.0 else nn.Identity() + self.gamma_v = nn.Parameter(init_values * torch.ones((v_dim)), requires_grad=True) + self.gamma_l = nn.Parameter(init_values * torch.ones((l_dim)), requires_grad=True) + + def forward(self, v, l, attention_mask_v=None, attention_mask_l=None): + v = self.layer_norm_v(v) + l = self.layer_norm_l(l) + delta_v, delta_l = self.attn( + v, l, attention_mask_v=attention_mask_v, attention_mask_l=attention_mask_l + ) + # v, l = v + delta_v, l + delta_l + v = v + self.drop_path(self.gamma_v * delta_v) + l = l + self.drop_path(self.gamma_l * delta_l) + return v, l + + # def forward(self, v:List[torch.Tensor], l, attention_mask_v=None, attention_mask_l=None) diff --git a/models/GroundingDINO/groundingdino.py b/models/GroundingDINO/groundingdino.py new file mode 100644 index 0000000000000000000000000000000000000000..e988982da8b9a57a48989add2a13e27b6d67c4bb --- /dev/null +++ b/models/GroundingDINO/groundingdino.py @@ -0,0 +1,1305 @@ +# ------------------------------------------------------------------------ +# Grounding DINO +# url: https://github.com/IDEA-Research/GroundingDINO +# Copyright (c) 2023 IDEA. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------ +# Conditional DETR model and criterion classes. +# Copyright (c) 2021 Microsoft. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------ +# Modified from DETR (https://github.com/facebookresearch/detr) +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +# ------------------------------------------------------------------------ +# Modified from Deformable DETR (https://github.com/fundamentalvision/Deformable-DETR) +# Copyright (c) 2020 SenseTime. All Rights Reserved. +# ------------------------------------------------------------------------ +import copy +from typing import List +import torchvision.transforms.functional as vis_F +from torchvision.transforms import InterpolationMode +import torch +import torch.nn.functional as F +from torch import nn +from torchvision.ops.boxes import nms +from torchvision.ops import roi_align +from transformers import ( + AutoTokenizer, + BertModel, + BertTokenizer, + RobertaModel, + RobertaTokenizerFast, +) + +from groundingdino.util import box_ops, get_tokenlizer +from groundingdino.util.misc import ( + NestedTensor, + accuracy, + get_world_size, + interpolate, + inverse_sigmoid, + is_dist_avail_and_initialized, + nested_tensor_from_tensor_list, +) +from groundingdino.util.utils import get_phrases_from_posmap +from groundingdino.util.visualizer import COCOVisualizer +from groundingdino.util.vl_utils import create_positive_map_from_span + +from ..registry import MODULE_BUILD_FUNCS +from .backbone import build_backbone +from .bertwarper import ( + BertModelWarper, + generate_masks_with_special_tokens, + generate_masks_with_special_tokens_and_transfer_map, +) +from .transformer import build_transformer +from .utils import MLP, ContrastiveEmbed, sigmoid_focal_loss + +from .matcher import build_matcher +import numpy as np +import matplotlib.pyplot as plt +from matplotlib.patches import Rectangle +from groundingdino.util.visualizer import renorm + + +def numpy_2_cv2(np_img): + if np.min(np_img) < 0: + raise Exception("image min is less than 0. Img min: " + str(np.min(np_img))) + if np.max(np_img) > 1: + raise Exception("image max is greater than 1. Img max: " + str(np.max(np_img))) + np_img = (np_img * 255).astype(np.uint8) + # Need to somehow ensure image is in RGB format. Note this line shows up in SAM demo: image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) + cv2_image = np.asarray(np_img) + return cv2_image + + +def vis_exemps(image, exemp, f_name): + plt.imshow(image) + plt.gca().add_patch( + Rectangle( + (exemp[0], exemp[1]), + exemp[2] - exemp[0], + exemp[3] - exemp[1], + edgecolor="red", + facecolor="none", + lw=1, + ) + ) + plt.savefig(f_name) + plt.close() + + +class GroundingDINO(nn.Module): + """This is the Cross-Attention Detector module that performs object detection""" + + def __init__( + self, + backbone, + transformer, + num_queries, + aux_loss=False, + iter_update=False, + query_dim=2, + num_feature_levels=1, + nheads=8, + # two stage + two_stage_type="no", # ['no', 'standard'] + dec_pred_bbox_embed_share=True, + two_stage_class_embed_share=True, + two_stage_bbox_embed_share=True, + num_patterns=0, + dn_number=100, + dn_box_noise_scale=0.4, + dn_label_noise_ratio=0.5, + dn_labelbook_size=100, + text_encoder_type="bert-base-uncased", + sub_sentence_present=True, + max_text_len=256, + ): + """Initializes the model. + Parameters: + backbone: torch module of the backbone to be used. See backbone.py + transformer: torch module of the transformer architecture. See transformer.py + num_queries: number of object queries, ie detection slot. This is the maximal number of objects + Conditional DETR can detect in a single image. For COCO, we recommend 100 queries. + aux_loss: True if auxiliary decoding losses (loss at each decoder layer) are to be used. + """ + super().__init__() + self.num_queries = num_queries + self.transformer = transformer + self.hidden_dim = hidden_dim = transformer.d_model + self.num_feature_levels = num_feature_levels + self.nheads = nheads + self.max_text_len = max_text_len + self.sub_sentence_present = sub_sentence_present + + # setting query dim + self.query_dim = query_dim + assert query_dim == 4 + + # visual exemplar cropping + self.feature_map_proj = nn.Conv2d((256 + 512 + 1024), hidden_dim, kernel_size=1) + + # for dn training + self.num_patterns = num_patterns + self.dn_number = dn_number + self.dn_box_noise_scale = dn_box_noise_scale + self.dn_label_noise_ratio = dn_label_noise_ratio + self.dn_labelbook_size = dn_labelbook_size + + # bert + self.tokenizer = get_tokenlizer.get_tokenlizer(text_encoder_type) + self.bert = get_tokenlizer.get_pretrained_language_model(text_encoder_type) + self.bert.pooler.dense.weight.requires_grad_(False) + self.bert.pooler.dense.bias.requires_grad_(False) + self.bert = BertModelWarper(bert_model=self.bert) + + self.feat_map = nn.Linear( + self.bert.config.hidden_size, self.hidden_dim, bias=True + ) + nn.init.constant_(self.feat_map.bias.data, 0) + nn.init.xavier_uniform_(self.feat_map.weight.data) + # freeze + + # special tokens + self.specical_tokens = self.tokenizer.convert_tokens_to_ids( + ["[CLS]", "[SEP]", ".", "?"] + ) + + # prepare input projection layers + if num_feature_levels > 1: + num_backbone_outs = len(backbone.num_channels) + input_proj_list = [] + for _ in range(num_backbone_outs): + in_channels = backbone.num_channels[_] + input_proj_list.append( + nn.Sequential( + nn.Conv2d(in_channels, hidden_dim, kernel_size=1), + nn.GroupNorm(32, hidden_dim), + ) + ) + for _ in range(num_feature_levels - num_backbone_outs): + input_proj_list.append( + nn.Sequential( + nn.Conv2d( + in_channels, hidden_dim, kernel_size=3, stride=2, padding=1 + ), + nn.GroupNorm(32, hidden_dim), + ) + ) + in_channels = hidden_dim + self.input_proj = nn.ModuleList(input_proj_list) + else: + assert ( + two_stage_type == "no" + ), "two_stage_type should be no if num_feature_levels=1 !!!" + self.input_proj = nn.ModuleList( + [ + nn.Sequential( + nn.Conv2d(backbone.num_channels[-1], hidden_dim, kernel_size=1), + nn.GroupNorm(32, hidden_dim), + ) + ] + ) + + self.backbone = backbone + self.aux_loss = aux_loss + self.box_pred_damping = box_pred_damping = None + + self.iter_update = iter_update + assert iter_update, "Why not iter_update?" + + # prepare pred layers + self.dec_pred_bbox_embed_share = dec_pred_bbox_embed_share + # prepare class & box embed + _class_embed = ContrastiveEmbed() + + _bbox_embed = MLP(hidden_dim, hidden_dim, 4, 3) + nn.init.constant_(_bbox_embed.layers[-1].weight.data, 0) + nn.init.constant_(_bbox_embed.layers[-1].bias.data, 0) + + if dec_pred_bbox_embed_share: + box_embed_layerlist = [ + _bbox_embed for i in range(transformer.num_decoder_layers) + ] + else: + box_embed_layerlist = [ + copy.deepcopy(_bbox_embed) + for i in range(transformer.num_decoder_layers) + ] + class_embed_layerlist = [ + _class_embed for i in range(transformer.num_decoder_layers) + ] + self.bbox_embed = nn.ModuleList(box_embed_layerlist) + self.class_embed = nn.ModuleList(class_embed_layerlist) + self.transformer.decoder.bbox_embed = self.bbox_embed + self.transformer.decoder.class_embed = self.class_embed + + # two stage + self.two_stage_type = two_stage_type + assert two_stage_type in [ + "no", + "standard", + ], "unknown param {} of two_stage_type".format(two_stage_type) + if two_stage_type != "no": + if two_stage_bbox_embed_share: + assert dec_pred_bbox_embed_share + self.transformer.enc_out_bbox_embed = _bbox_embed + else: + self.transformer.enc_out_bbox_embed = copy.deepcopy(_bbox_embed) + + if two_stage_class_embed_share: + assert dec_pred_bbox_embed_share + self.transformer.enc_out_class_embed = _class_embed + else: + self.transformer.enc_out_class_embed = copy.deepcopy(_class_embed) + + self.refpoint_embed = None + + self._reset_parameters() + + def _reset_parameters(self): + # init input_proj + for proj in self.input_proj: + nn.init.xavier_uniform_(proj[0].weight, gain=1) + nn.init.constant_(proj[0].bias, 0) + + def init_ref_points(self, use_num_queries): + self.refpoint_embed = nn.Embedding(use_num_queries, self.query_dim) + + def add_exemplar_tokens(self, tokenized, text_dict, exemplar_tokens, labels): + input_ids = tokenized["input_ids"] + + device = input_ids.device + new_input_ids = [] + encoded_text = text_dict["encoded_text"] + new_encoded_text = [] + text_token_mask = text_dict["text_token_mask"] + new_text_token_mask = [] + position_ids = text_dict["position_ids"] + text_self_attention_masks = text_dict["text_self_attention_masks"] + + for sample_ind in range(len(labels)): + label = labels[sample_ind][0] + exemplars = exemplar_tokens[sample_ind] + label_count = -1 + assert len(input_ids[sample_ind]) == len(position_ids[sample_ind]) + for token_ind in range(len(input_ids[sample_ind])): + input_id = input_ids[sample_ind][token_ind] + if (input_id not in self.specical_tokens) and ( + token_ind == 0 + or (input_ids[sample_ind][token_ind - 1] in self.specical_tokens) + ): + label_count += 1 + if label_count == label: + # Get the index where to insert the exemplar tokens. + ind_to_insert_exemplar = token_ind + while ( + input_ids[sample_ind][ind_to_insert_exemplar] + not in self.specical_tokens + ): + ind_to_insert_exemplar += 1 + break + + # Handle no text case. + if label_count == -1: + ind_to_insert_exemplar = 1 + # * token indicates exemplar. + new_input_ids.append( + torch.cat( + [ + input_ids[sample_ind][:ind_to_insert_exemplar], + torch.tensor([1008] * exemplars.shape[0]).to(device), + input_ids[sample_ind][ind_to_insert_exemplar:], + ] + ) + ) + new_encoded_text.append( + torch.cat( + [ + encoded_text[sample_ind][:ind_to_insert_exemplar, :], + exemplars, + encoded_text[sample_ind][ind_to_insert_exemplar:, :], + ] + ) + ) + new_text_token_mask.append( + torch.full((len(new_input_ids[sample_ind]),), True).to(device) + ) + + tokenized["input_ids"] = torch.stack(new_input_ids) + print(tokenized["input_ids"]) + + ( + text_self_attention_masks, + position_ids, + _, + ) = generate_masks_with_special_tokens_and_transfer_map( + tokenized, self.specical_tokens, None + ) + + return { + "encoded_text": torch.stack(new_encoded_text), + "text_token_mask": torch.stack(new_text_token_mask), + "position_ids": position_ids, + "text_self_attention_masks": text_self_attention_masks, + } + + def combine_features(self, features): + (bs, c, h, w) = ( + features[0].decompose()[0].shape[-4], + features[0].decompose()[0].shape[-3], + features[0].decompose()[0].shape[-2], + features[0].decompose()[0].shape[-1], + ) + + x = torch.cat( + [ + F.interpolate( + feat.decompose()[0], + size=(h, w), + mode="bilinear", + align_corners=True, + ) + for feat in features + ], + dim=1, + ) + + x = self.feature_map_proj(x) + + return x + + def forward( + self, + samples: NestedTensor, + exemplar_images: NestedTensor, + exemplars: List, + labels, + targets: List = None, + cropped=False, + orig_img=None, + crop_width=0, + crop_height=0, + **kw, + ): + """The forward expects a NestedTensor, which consists of: + - samples.tensor: batched images, of shape [batch_size x 3 x H x W] + - samples.mask: a binary mask of shape [batch_size x H x W], containing 1 on padded pixels + + It returns a dict with the following elements: + - "pred_logits": the classification logits (including no-object) for all queries. + Shape= [batch_size x num_queries x num_classes] + - "pred_boxes": The normalized boxes coordinates for all queries, represented as + (center_x, center_y, width, height). These values are normalized in [0, 1], + relative to the size of each individual image (disregarding possible padding). + See PostProcess for information on how to retrieve the unnormalized bounding box. + - "aux_outputs": Optional, only returned when auxilary losses are activated. It is a list of + dictionnaries containing the two above keys for each decoder layer. + """ + + if targets is None: + captions = kw["captions"] + else: + captions = [t["caption"] for t in targets] + + # encoder texts + + tokenized = self.tokenizer(captions, padding="longest", return_tensors="pt").to( + samples.device + ) + + one_hot_token = tokenized + + ( + text_self_attention_masks, + position_ids, + cate_to_token_mask_list, + ) = generate_masks_with_special_tokens_and_transfer_map( + tokenized, self.specical_tokens, self.tokenizer + ) + + if text_self_attention_masks.shape[1] > self.max_text_len: + text_self_attention_masks = text_self_attention_masks[ + :, : self.max_text_len, : self.max_text_len + ] + position_ids = position_ids[:, : self.max_text_len] + tokenized["input_ids"] = tokenized["input_ids"][:, : self.max_text_len] + tokenized["attention_mask"] = tokenized["attention_mask"][ + :, : self.max_text_len + ] + tokenized["token_type_ids"] = tokenized["token_type_ids"][ + :, : self.max_text_len + ] + + # extract text embeddings + if self.sub_sentence_present: + tokenized_for_encoder = { + k: v for k, v in tokenized.items() if k != "attention_mask" + } + tokenized_for_encoder["attention_mask"] = text_self_attention_masks + tokenized_for_encoder["position_ids"] = position_ids + else: + tokenized_for_encoder = tokenized + + bert_output = self.bert(**tokenized_for_encoder) # bs, 195, 768 + + encoded_text = self.feat_map( + bert_output["last_hidden_state"] + ) # bs, 195, d_model + text_token_mask = tokenized.attention_mask.bool() # bs, 195 + # text_token_mask: True for nomask, False for mask + # text_self_attention_masks: True for nomask, False for mask + + if encoded_text.shape[1] > self.max_text_len: + encoded_text = encoded_text[:, : self.max_text_len, :] + text_token_mask = text_token_mask[:, : self.max_text_len] + position_ids = position_ids[:, : self.max_text_len] + text_self_attention_masks = text_self_attention_masks[ + :, : self.max_text_len, : self.max_text_len + ] + + text_dict = { + "encoded_text": encoded_text, # bs, 195, d_model + "text_token_mask": text_token_mask, # bs, 195 + "position_ids": position_ids, # bs, 195 + "text_self_attention_masks": text_self_attention_masks, # bs, 195,195 + } + + if isinstance(samples, (list, torch.Tensor)): + samples = nested_tensor_from_tensor_list(samples) + + if not cropped: + features, poss = self.backbone(samples) + features_exemp, _ = self.backbone(exemplar_images) + combined_features = self.combine_features(features_exemp) + # Get visual exemplar tokens. + bs = len(exemplars) + num_exemplars = exemplars[0].shape[0] + print(exemplars) + print(num_exemplars) + if num_exemplars > 0: + exemplar_tokens = ( + roi_align( + combined_features, + boxes=exemplars, + output_size=(1, 1), + spatial_scale=(1 / 8), + aligned=True, + ) + .squeeze(-1) + .squeeze(-1) + .reshape(bs, num_exemplars, -1) + ) + else: + exemplar_tokens = None + + else: + features, poss = self.backbone(samples) + (h, w) = ( + samples.decompose()[0][0].shape[1], + samples.decompose()[0][0].shape[2], + ) + (orig_img_h, orig_img_w) = orig_img.shape[1], orig_img.shape[2] + bs = len(samples.decompose()[0]) + + exemp_imgs = [] + new_exemplars = [] + ind = 0 + for exemp in exemplars[0]: + center_x = (exemp[0] + exemp[2]) / 2 + center_y = (exemp[1] + exemp[3]) / 2 + start_x = max(int(center_x - crop_width / 2), 0) + end_x = min(int(center_x + crop_width / 2), orig_img_w) + start_y = max(int(center_y - crop_height / 2), 0) + end_y = min(int(center_y + crop_height / 2), orig_img_h) + scale_x = w / (end_x - start_x) + scale_y = h / (end_y - start_y) + exemp_imgs.append( + vis_F.resize( + orig_img[:, start_y:end_y, start_x:end_x], + (h, w), + interpolation=InterpolationMode.BICUBIC, + ) + ) + new_exemplars.append( + [ + (exemp[0] - start_x) * scale_x, + (exemp[1] - start_y) * scale_y, + (exemp[2] - start_x) * scale_x, + (exemp[3] - start_y) * scale_y, + ] + ) + + vis_exemps( + renorm(exemp_imgs[-1].cpu()).permute(1, 2, 0).numpy(), + [coord.item() for coord in new_exemplars[-1]], + str(ind) + ".jpg", + ) + vis_exemps( + renorm(orig_img.cpu()).permute(1, 2, 0).numpy(), + [coord.item() for coord in exemplars[0][ind]], + "orig-" + str(ind) + ".jpg", + ) + ind += 1 + + exemp_imgs = nested_tensor_from_tensor_list(exemp_imgs) + features_exemp, _ = self.backbone(exemp_imgs) + combined_features = self.combine_features(features_exemp) + new_exemplars = [ + torch.tensor(exemp).unsqueeze(0).cuda() for exemp in new_exemplars + ] + + # Get visual exemplar tokens. + exemplar_tokens = ( + roi_align( + combined_features, + boxes=new_exemplars, + output_size=(1, 1), + spatial_scale=(1 / 8), + aligned=True, + ) + .squeeze(-1) + .squeeze(-1) + .reshape(3, 256) + ) + + exemplar_tokens = torch.stack([exemplar_tokens] * bs) + + if exemplar_tokens is not None: + text_dict = self.add_exemplar_tokens( + tokenized, text_dict, exemplar_tokens, labels + ) + + srcs = [] + masks = [] + for l, feat in enumerate(features): + src, mask = feat.decompose() + srcs.append(self.input_proj[l](src)) + masks.append(mask) + assert mask is not None + if self.num_feature_levels > len(srcs): + _len_srcs = len(srcs) + for l in range(_len_srcs, self.num_feature_levels): + if l == _len_srcs: + src = self.input_proj[l](features[-1].tensors) + else: + src = self.input_proj[l](srcs[-1]) + m = samples.mask + mask = F.interpolate(m[None].float(), size=src.shape[-2:]).to( + torch.bool + )[0] + pos_l = self.backbone[1](NestedTensor(src, mask)).to(src.dtype) + srcs.append(src) + masks.append(mask) + poss.append(pos_l) + + input_query_bbox = input_query_label = attn_mask = dn_meta = None + hs, reference, hs_enc, ref_enc, init_box_proposal = self.transformer( + srcs, masks, input_query_bbox, poss, input_query_label, attn_mask, text_dict + ) + + # deformable-detr-like anchor update + outputs_coord_list = [] + for dec_lid, (layer_ref_sig, layer_bbox_embed, layer_hs) in enumerate( + zip(reference[:-1], self.bbox_embed, hs) + ): + layer_delta_unsig = layer_bbox_embed(layer_hs) + layer_outputs_unsig = layer_delta_unsig + inverse_sigmoid(layer_ref_sig) + layer_outputs_unsig = layer_outputs_unsig.sigmoid() + outputs_coord_list.append(layer_outputs_unsig) + outputs_coord_list = torch.stack(outputs_coord_list) + + outputs_class = torch.stack( + [ + layer_cls_embed(layer_hs, text_dict) + for layer_cls_embed, layer_hs in zip(self.class_embed, hs) + ] + ) + + out = {"pred_logits": outputs_class[-1], "pred_boxes": outputs_coord_list[-1]} + + # Used to calculate losses + bs, len_td = text_dict["text_token_mask"].shape + out["text_mask"] = torch.zeros(bs, self.max_text_len, dtype=torch.bool).to( + samples.device + ) + for b in range(bs): + for j in range(len_td): + if text_dict["text_token_mask"][b][j] == True: + out["text_mask"][b][j] = True + + # for intermediate outputs + if self.aux_loss: + out["aux_outputs"] = self._set_aux_loss(outputs_class, outputs_coord_list) + out["token"] = one_hot_token + # # for encoder output + if hs_enc is not None: + # prepare intermediate outputs + interm_coord = ref_enc[-1] + interm_class = self.transformer.enc_out_class_embed(hs_enc[-1], text_dict) + out["interm_outputs"] = { + "pred_logits": interm_class, + "pred_boxes": interm_coord, + } + out["interm_outputs_for_matching_pre"] = { + "pred_logits": interm_class, + "pred_boxes": init_box_proposal, + } + + # outputs['pred_logits'].shape + # torch.Size([4, 900, 256]) + + # outputs['pred_boxes'].shape + # torch.Size([4, 900, 4]) + + # outputs['text_mask'].shape + # torch.Size([256]) + + # outputs['text_mask'] + + # outputs['aux_outputs'][0].keys() + # dict_keys(['pred_logits', 'pred_boxes', 'one_hot', 'text_mask']) + + # outputs['aux_outputs'][img_idx] + + # outputs['token'] + # + + # outputs['interm_outputs'].keys() + # dict_keys(['pred_logits', 'pred_boxes', 'one_hot', 'text_mask']) + + # outputs['interm_outputs_for_matching_pre'].keys() + # dict_keys(['pred_logits', 'pred_boxes']) + + # outputs['one_hot'].shape + # torch.Size([4, 900, 256]) + + return out + + @torch.jit.unused + def _set_aux_loss(self, outputs_class, outputs_coord): + # this is a workaround to make torchscript happy, as torchscript + # doesn't support dictionary with non-homogeneous values, such + # as a dict having both a Tensor and a list. + return [ + {"pred_logits": a, "pred_boxes": b} + for a, b in zip(outputs_class[:-1], outputs_coord[:-1]) + ] + + +class SetCriterion(nn.Module): + def __init__(self, matcher, weight_dict, focal_alpha, focal_gamma, losses): + """Create the criterion. + Parameters: + matcher: module able to compute a matching between targets and proposals + weight_dict: dict containing as key the names of the losses and as values their relative weight. + losses: list of all the losses to be applied. See get_loss for list of available losses. + focal_alpha: alpha in Focal Loss + """ + super().__init__() + self.matcher = matcher + self.weight_dict = weight_dict + self.losses = losses + self.focal_alpha = focal_alpha + self.focal_gamma = focal_gamma + + @torch.no_grad() + def loss_cardinality(self, outputs, targets, indices, num_boxes): + """Compute the cardinality error, ie the absolute error in the number of predicted non-empty boxes + This is not really a loss, it is intended for logging purposes only. It doesn't propagate gradients + """ + + pred_logits = outputs["pred_logits"] + device = pred_logits.device + tgt_lengths = torch.as_tensor( + [len(v["labels"]) for v in targets], device=device + ) + # Count the number of predictions that are NOT "no-object" (which is the last class) + card_pred = (pred_logits.argmax(-1) != pred_logits.shape[-1] - 1).sum(1) + card_err = F.l1_loss(card_pred.float(), tgt_lengths.float()) + losses = {"cardinality_error": card_err} + return losses + + def loss_boxes(self, outputs, targets, indices, num_boxes): + """Compute the losses related to the bounding boxes, the L1 regression loss and the GIoU loss + targets dicts must contain the key "boxes" containing a tensor of dim [nb_target_boxes, 4] + The target boxes are expected in format (center_x, center_y, w, h), normalized by the image size. + """ + assert "pred_boxes" in outputs + idx = self._get_src_permutation_idx(indices) + src_boxes = outputs["pred_boxes"][idx] + target_boxes = torch.cat( + [t["boxes"][i] for t, (_, i) in zip(targets, indices)], dim=0 + ) + + loss_bbox = F.l1_loss(src_boxes[:, :2], target_boxes[:, :2], reduction="none") + + losses = {} + losses["loss_bbox"] = loss_bbox.sum() / num_boxes + + loss_giou = 1 - torch.diag( + box_ops.generalized_box_iou( + box_ops.box_cxcywh_to_xyxy(src_boxes), + box_ops.box_cxcywh_to_xyxy(target_boxes), + ) + ) + losses["loss_giou"] = loss_giou.sum() / num_boxes + + # calculate the x,y and h,w loss + with torch.no_grad(): + losses["loss_xy"] = loss_bbox[..., :2].sum() / num_boxes + losses["loss_hw"] = loss_bbox[..., 2:].sum() / num_boxes + + return losses + + def token_sigmoid_binary_focal_loss(self, outputs, targets, indices, num_boxes): + pred_logits = outputs["pred_logits"] + new_targets = outputs["one_hot"].to(pred_logits.device) + text_mask = outputs["text_mask"] + + assert new_targets.dim() == 3 + assert pred_logits.dim() == 3 # batch x from x to + + bs, n, _ = pred_logits.shape + alpha = self.focal_alpha + gamma = self.focal_gamma + if text_mask is not None: + # ODVG: each sample has different mask + text_mask = text_mask.repeat(1, pred_logits.size(1)).view( + outputs["text_mask"].shape[0], -1, outputs["text_mask"].shape[1] + ) + pred_logits = torch.masked_select(pred_logits, text_mask) + new_targets = torch.masked_select(new_targets, text_mask) + + new_targets = new_targets.float() + p = torch.sigmoid(pred_logits) + ce_loss = F.binary_cross_entropy_with_logits( + pred_logits, new_targets, reduction="none" + ) + p_t = p * new_targets + (1 - p) * (1 - new_targets) + loss = ce_loss * ((1 - p_t) ** gamma) + + if alpha >= 0: + alpha_t = alpha * new_targets + (1 - alpha) * (1 - new_targets) + loss = alpha_t * loss + + total_num_pos = 0 + for batch_indices in indices: + total_num_pos += len(batch_indices[0]) + num_pos_avg_per_gpu = max(total_num_pos, 1.0) + loss = loss.sum() / num_pos_avg_per_gpu + + losses = {"loss_ce": loss} + return losses + + def _get_src_permutation_idx(self, indices): + # permute predictions following indices + batch_idx = torch.cat( + [torch.full_like(src, i) for i, (src, _) in enumerate(indices)] + ) + src_idx = torch.cat([src for (src, _) in indices]) + return batch_idx, src_idx + + def _get_tgt_permutation_idx(self, indices): + # permute targets following indices + batch_idx = torch.cat( + [torch.full_like(tgt, i) for i, (_, tgt) in enumerate(indices)] + ) + tgt_idx = torch.cat([tgt for (_, tgt) in indices]) + return batch_idx, tgt_idx + + def get_loss(self, loss, outputs, targets, indices, num_boxes, **kwargs): + loss_map = { + "labels": self.token_sigmoid_binary_focal_loss, + "cardinality": self.loss_cardinality, + "boxes": self.loss_boxes, + } + assert loss in loss_map, f"do you really want to compute {loss} loss?" + return loss_map[loss](outputs, targets, indices, num_boxes, **kwargs) + + def forward(self, outputs, targets, cat_list, caption, return_indices=False): + """This performs the loss computation. + Parameters: + outputs: dict of tensors, see the output specification of the model for the format + targets: list of dicts, such that len(targets) == batch_size. + The expected keys in each dict depends on the losses applied, see each loss' doc + + return_indices: used for vis. if True, the layer0-5 indices will be returned as well. + """ + device = next(iter(outputs.values())).device + one_hot = torch.zeros( + outputs["pred_logits"].size(), dtype=torch.int64 + ) # torch.Size([bs, 900, 256]) + token = outputs["token"] + + label_map_list = [] + indices = [] + for j in range(len(cat_list)): # bs + label_map = [] + for i in range(len(cat_list[j])): + label_id = torch.tensor([i]) + per_label = create_positive_map_exemplar( + token["input_ids"][j], label_id, [101, 102, 1012, 1029] + ) + label_map.append(per_label) + label_map = torch.stack(label_map, dim=0).squeeze(1) + + label_map_list.append(label_map) + for j in range(len(cat_list)): # bs + for_match = { + "pred_logits": outputs["pred_logits"][j].unsqueeze(0), + "pred_boxes": outputs["pred_boxes"][j].unsqueeze(0), + } + + inds = self.matcher(for_match, [targets[j]], label_map_list[j]) + indices.extend(inds) + # indices : A list of size batch_size, containing tuples of (index_i, index_j) where: + # - index_i is the indices of the selected predictions (in order) + # - index_j is the indices of the corresponding selected targets (in order) + + # import pdb; pdb.set_trace() + tgt_ids = [v["labels"].cpu() for v in targets] + # len(tgt_ids) == bs + for i in range(len(indices)): + tgt_ids[i] = tgt_ids[i][indices[i][1]] + one_hot[i, indices[i][0]] = label_map_list[i][tgt_ids[i]].to(torch.long) + outputs["one_hot"] = one_hot + if return_indices: + indices0_copy = indices + indices_list = [] + + # Compute the average number of target boxes accross all nodes, for normalization purposes + num_boxes_list = [len(t["labels"]) for t in targets] + num_boxes = sum(num_boxes_list) + num_boxes = torch.as_tensor([num_boxes], dtype=torch.float, device=device) + if is_dist_avail_and_initialized(): + torch.distributed.all_reduce(num_boxes) + num_boxes = torch.clamp(num_boxes / get_world_size(), min=1).item() + + # Compute all the requested losses + losses = {} + for loss in self.losses: + losses.update(self.get_loss(loss, outputs, targets, indices, num_boxes)) + + # In case of auxiliary losses, we repeat this process with the output of each intermediate layer. + if "aux_outputs" in outputs: + for idx, aux_outputs in enumerate(outputs["aux_outputs"]): + indices = [] + for j in range(len(cat_list)): # bs + aux_output_single = { + "pred_logits": aux_outputs["pred_logits"][j].unsqueeze(0), + "pred_boxes": aux_outputs["pred_boxes"][j].unsqueeze(0), + } + inds = self.matcher( + aux_output_single, [targets[j]], label_map_list[j] + ) + indices.extend(inds) + one_hot_aux = torch.zeros( + outputs["pred_logits"].size(), dtype=torch.int64 + ) + tgt_ids = [v["labels"].cpu() for v in targets] + for i in range(len(indices)): + tgt_ids[i] = tgt_ids[i][indices[i][1]] + one_hot_aux[i, indices[i][0]] = label_map_list[i][tgt_ids[i]].to( + torch.long + ) + aux_outputs["one_hot"] = one_hot_aux + aux_outputs["text_mask"] = outputs["text_mask"] + if return_indices: + indices_list.append(indices) + for loss in self.losses: + kwargs = {} + l_dict = self.get_loss( + loss, aux_outputs, targets, indices, num_boxes, **kwargs + ) + l_dict = {k + f"_{idx}": v for k, v in l_dict.items()} + losses.update(l_dict) + + # interm_outputs loss + if "interm_outputs" in outputs: + interm_outputs = outputs["interm_outputs"] + indices = [] + for j in range(len(cat_list)): # bs + interm_output_single = { + "pred_logits": interm_outputs["pred_logits"][j].unsqueeze(0), + "pred_boxes": interm_outputs["pred_boxes"][j].unsqueeze(0), + } + inds = self.matcher( + interm_output_single, [targets[j]], label_map_list[j] + ) + indices.extend(inds) + one_hot_aux = torch.zeros(outputs["pred_logits"].size(), dtype=torch.int64) + tgt_ids = [v["labels"].cpu() for v in targets] + for i in range(len(indices)): + tgt_ids[i] = tgt_ids[i][indices[i][1]] + one_hot_aux[i, indices[i][0]] = label_map_list[i][tgt_ids[i]].to( + torch.long + ) + interm_outputs["one_hot"] = one_hot_aux + interm_outputs["text_mask"] = outputs["text_mask"] + if return_indices: + indices_list.append(indices) + for loss in self.losses: + kwargs = {} + l_dict = self.get_loss( + loss, interm_outputs, targets, indices, num_boxes, **kwargs + ) + l_dict = {k + f"_interm": v for k, v in l_dict.items()} + losses.update(l_dict) + + if return_indices: + indices_list.append(indices0_copy) + return losses, indices_list + + return losses + + +class PostProcess(nn.Module): + """This module converts the model's output into the format expected by the coco api""" + + def __init__( + self, + num_select=100, + text_encoder_type="text_encoder_type", + nms_iou_threshold=-1, + use_coco_eval=False, + args=None, + ) -> None: + super().__init__() + self.num_select = num_select + self.tokenizer = get_tokenlizer.get_tokenlizer(text_encoder_type) + if args.use_coco_eval: + from pycocotools.coco import COCO + + coco = COCO(args.coco_val_path) + category_dict = coco.loadCats(coco.getCatIds()) + cat_list = [item["name"] for item in category_dict] + else: + cat_list = args.label_list + caption = " . ".join(cat_list) + " ." + tokenized = self.tokenizer(caption, padding="longest", return_tensors="pt") + label_list = torch.arange(len(cat_list)) + pos_map = create_positive_map(tokenized, label_list, cat_list, caption) + # build a mapping from label_id to pos_map + if args.use_coco_eval: + id_map = { + 0: 1, + 1: 2, + 2: 3, + 3: 4, + 4: 5, + 5: 6, + 6: 7, + 7: 8, + 8: 9, + 9: 10, + 10: 11, + 11: 13, + 12: 14, + 13: 15, + 14: 16, + 15: 17, + 16: 18, + 17: 19, + 18: 20, + 19: 21, + 20: 22, + 21: 23, + 22: 24, + 23: 25, + 24: 27, + 25: 28, + 26: 31, + 27: 32, + 28: 33, + 29: 34, + 30: 35, + 31: 36, + 32: 37, + 33: 38, + 34: 39, + 35: 40, + 36: 41, + 37: 42, + 38: 43, + 39: 44, + 40: 46, + 41: 47, + 42: 48, + 43: 49, + 44: 50, + 45: 51, + 46: 52, + 47: 53, + 48: 54, + 49: 55, + 50: 56, + 51: 57, + 52: 58, + 53: 59, + 54: 60, + 55: 61, + 56: 62, + 57: 63, + 58: 64, + 59: 65, + 60: 67, + 61: 70, + 62: 72, + 63: 73, + 64: 74, + 65: 75, + 66: 76, + 67: 77, + 68: 78, + 69: 79, + 70: 80, + 71: 81, + 72: 82, + 73: 84, + 74: 85, + 75: 86, + 76: 87, + 77: 88, + 78: 89, + 79: 90, + } + new_pos_map = torch.zeros((91, 256)) + for k, v in id_map.items(): + new_pos_map[v] = pos_map[k] + pos_map = new_pos_map + + self.nms_iou_threshold = nms_iou_threshold + self.positive_map = pos_map + + @torch.no_grad() + def forward(self, outputs, target_sizes, not_to_xyxy=False, test=False): + """Perform the computation + Parameters: + outputs: raw outputs of the model + target_sizes: tensor of dimension [batch_size x 2] containing the size of each images of the batch + For evaluation, this must be the original image size (before any data augmentation) + For visualization, this should be the image size after data augment, but before padding + """ + num_select = self.num_select + out_logits, out_bbox = outputs["pred_logits"], outputs["pred_boxes"] + + prob_to_token = out_logits.sigmoid() + pos_maps = self.positive_map.to(prob_to_token.device) + for label_ind in range(len(pos_maps)): + if pos_maps[label_ind].sum() != 0: + pos_maps[label_ind] = pos_maps[label_ind] / pos_maps[label_ind].sum() + + prob_to_label = prob_to_token @ pos_maps.T + + assert len(out_logits) == len(target_sizes) + assert target_sizes.shape[1] == 2 + + prob = prob_to_label + topk_values, topk_indexes = torch.topk( + prob.view(prob.shape[0], -1), num_select, dim=1 + ) + scores = topk_values + topk_boxes = torch.div(topk_indexes, prob.shape[2], rounding_mode="trunc") + labels = topk_indexes % prob.shape[2] + if not_to_xyxy: + boxes = out_bbox + else: + boxes = box_ops.box_cxcywh_to_xyxy(out_bbox) + + # if test: + # assert not not_to_xyxy + # boxes[:,:,2:] = boxes[:,:,2:] - boxes[:,:,:2] + boxes = torch.gather(boxes, 1, topk_boxes.unsqueeze(-1).repeat(1, 1, 4)) + + # and from relative [0, 1] to absolute [0, height] coordinates + img_h, img_w = target_sizes.unbind(1) + scale_fct = torch.stack([img_w, img_h, img_w, img_h], dim=1) + boxes = boxes * scale_fct[:, None, :] + + if self.nms_iou_threshold > 0: + item_indices = [ + nms(b, s, iou_threshold=self.nms_iou_threshold) + for b, s in zip(boxes, scores) + ] + + results = [ + {"scores": s[i], "labels": l[i], "boxes": b[i]} + for s, l, b, i in zip(scores, labels, boxes, item_indices) + ] + else: + results = [ + {"scores": s, "labels": l, "boxes": b} + for s, l, b in zip(scores, labels, boxes) + ] + results = [ + {"scores": s, "labels": l, "boxes": b} + for s, l, b in zip(scores, labels, boxes) + ] + return results + + +@MODULE_BUILD_FUNCS.registe_with_name(module_name="groundingdino") +def build_groundingdino(args): + device = torch.device(args.device) + backbone = build_backbone(args) + transformer = build_transformer(args) + + dn_labelbook_size = args.dn_labelbook_size + dec_pred_bbox_embed_share = args.dec_pred_bbox_embed_share + sub_sentence_present = args.sub_sentence_present + + model = GroundingDINO( + backbone, + transformer, + num_queries=args.num_queries, + aux_loss=args.aux_loss, + iter_update=True, + query_dim=4, + num_feature_levels=args.num_feature_levels, + nheads=args.nheads, + dec_pred_bbox_embed_share=dec_pred_bbox_embed_share, + two_stage_type=args.two_stage_type, + two_stage_bbox_embed_share=args.two_stage_bbox_embed_share, + two_stage_class_embed_share=args.two_stage_class_embed_share, + num_patterns=args.num_patterns, + dn_number=0, + dn_box_noise_scale=args.dn_box_noise_scale, + dn_label_noise_ratio=args.dn_label_noise_ratio, + dn_labelbook_size=dn_labelbook_size, + text_encoder_type=args.text_encoder_type, + sub_sentence_present=sub_sentence_present, + max_text_len=args.max_text_len, + ) + + matcher = build_matcher(args) + + # prepare weight dict + weight_dict = {"loss_ce": args.cls_loss_coef, "loss_bbox": args.bbox_loss_coef} + weight_dict["loss_giou"] = args.giou_loss_coef + clean_weight_dict_wo_dn = copy.deepcopy(weight_dict) + + clean_weight_dict = copy.deepcopy(weight_dict) + + # TODO this is a hack + if args.aux_loss: + aux_weight_dict = {} + for i in range(args.dec_layers - 1): + aux_weight_dict.update( + {k + f"_{i}": v for k, v in clean_weight_dict.items()} + ) + weight_dict.update(aux_weight_dict) + + if args.two_stage_type != "no": + interm_weight_dict = {} + try: + no_interm_box_loss = args.no_interm_box_loss + except: + no_interm_box_loss = False + _coeff_weight_dict = { + "loss_ce": 1.0, + "loss_bbox": 1.0 if not no_interm_box_loss else 0.0, + "loss_giou": 1.0 if not no_interm_box_loss else 0.0, + } + try: + interm_loss_coef = args.interm_loss_coef + except: + interm_loss_coef = 1.0 + interm_weight_dict.update( + { + k + f"_interm": v * interm_loss_coef * _coeff_weight_dict[k] + for k, v in clean_weight_dict_wo_dn.items() + } + ) + weight_dict.update(interm_weight_dict) + + # losses = ['labels', 'boxes', 'cardinality'] + losses = ["labels", "boxes"] + + criterion = SetCriterion( + matcher=matcher, + weight_dict=weight_dict, + focal_alpha=args.focal_alpha, + focal_gamma=args.focal_gamma, + losses=losses, + ) + criterion.to(device) + postprocessors = { + "bbox": PostProcess( + num_select=args.num_select, + text_encoder_type=args.text_encoder_type, + nms_iou_threshold=args.nms_iou_threshold, + args=args, + ) + } + + return model, criterion, postprocessors + + +def create_positive_map(tokenized, tokens_positive, cat_list, caption): + """construct a map such that positive_map[i,j] = True iff box i is associated to token j""" + positive_map = torch.zeros((len(tokens_positive), 256), dtype=torch.float) + + for j, label in enumerate(tokens_positive): + start_ind = caption.find(cat_list[label]) + end_ind = start_ind + len(cat_list[label]) - 1 + beg_pos = tokenized.char_to_token(start_ind) + try: + end_pos = tokenized.char_to_token(end_ind) + except: + end_pos = None + if end_pos is None: + try: + end_pos = tokenized.char_to_token(end_ind - 1) + if end_pos is None: + end_pos = tokenized.char_to_token(end_ind - 2) + except: + end_pos = None + # except Exception as e: + # print("beg:", beg, "end:", end) + # print("token_positive:", tokens_positive) + # # print("beg_pos:", beg_pos, "end_pos:", end_pos) + # raise e + # if beg_pos is None: + # try: + # beg_pos = tokenized.char_to_token(beg + 1) + # if beg_pos is None: + # beg_pos = tokenized.char_to_token(beg + 2) + # except: + # beg_pos = None + # if end_pos is None: + # try: + # end_pos = tokenized.char_to_token(end - 2) + # if end_pos is None: + # end_pos = tokenized.char_to_token(end - 3) + # except: + # end_pos = None + if beg_pos is None or end_pos is None: + continue + if beg_pos < 0 or end_pos < 0: + continue + if beg_pos > end_pos: + continue + # assert beg_pos is not None and end_pos is not None + positive_map[j, beg_pos : end_pos + 1].fill_(1) + return positive_map + + +def create_positive_map_exemplar(input_ids, label, special_tokens): + tokens_positive = torch.zeros(256, dtype=torch.float) + count = -1 + for token_ind in range(len(input_ids)): + input_id = input_ids[token_ind] + if (input_id not in special_tokens) and ( + token_ind == 0 or (input_ids[token_ind - 1] in special_tokens) + ): + count += 1 + if count == label: + ind_to_insert_ones = token_ind + + while input_ids[ind_to_insert_ones] not in special_tokens: + tokens_positive[ind_to_insert_ones] = 1 + ind_to_insert_ones += 1 + break + return tokens_positive diff --git a/models/GroundingDINO/matcher.py b/models/GroundingDINO/matcher.py new file mode 100644 index 0000000000000000000000000000000000000000..daf43d8f1b92b2821773013dd0a36b8a969833f1 --- /dev/null +++ b/models/GroundingDINO/matcher.py @@ -0,0 +1,217 @@ +# ------------------------------------------------------------------------ +# DINO +# Copyright (c) 2022 IDEA. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------ +# Modules to compute the matching cost and solve the corresponding LSAP. +# Copyright (c) 2021 Microsoft. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------ +# Modified from DETR (https://github.com/facebookresearch/detr) +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +# ------------------------------------------------------------------------ +# Modified from Deformable DETR (https://github.com/fundamentalvision/Deformable-DETR) +# Copyright (c) 2020 SenseTime. All Rights Reserved. +# ------------------------------------------------------------------------ + + +import torch, os +from torch import nn +from scipy.optimize import linear_sum_assignment + +from util.box_ops import box_cxcywh_to_xyxy, generalized_box_iou + + +class HungarianMatcher(nn.Module): + """This class computes an assignment between the targets and the predictions of the network + For efficiency reasons, the targets don't include the no_object. Because of this, in general, + there are more predictions than targets. In this case, we do a 1-to-1 matching of the best predictions, + while the others are un-matched (and thus treated as non-objects). + """ + + def __init__(self, cost_class: float = 1, cost_bbox: float = 1, cost_giou: float = 1, focal_alpha = 0.25): + """Creates the matcher + Params: + cost_class: This is the relative weight of the classification error in the matching cost + cost_bbox: This is the relative weight of the L1 error of the bounding box coordinates in the matching cost + cost_giou: This is the relative weight of the giou loss of the bounding box in the matching cost + """ + super().__init__() + self.cost_class = cost_class + self.cost_bbox = cost_bbox + self.cost_giou = cost_giou + assert cost_class != 0 or cost_bbox != 0 or cost_giou != 0, "all costs cant be 0" + + self.focal_alpha = focal_alpha + + @torch.no_grad() + def forward(self, outputs, targets, label_map): + """ Performs the matching + Params: + outputs: This is a dict that contains at least these entries: + "pred_logits": Tensor of dim [batch_size, num_queries, num_classes] with the classification logits + "pred_boxes": Tensor of dim [batch_size, num_queries, 4] with the predicted box coordinates + targets: This is a list of targets (len(targets) = batch_size), where each target is a dict containing: + "labels": Tensor of dim [num_target_boxes] (where num_target_boxes is the number of ground-truth + objects in the target) containing the class labels + "boxes": Tensor of dim [num_target_boxes, 4] containing the target box coordinates + Returns: + A list of size batch_size, containing tuples of (index_i, index_j) where: + - index_i is the indices of the selected predictions (in order) + - index_j is the indices of the corresponding selected targets (in order) + For each batch element, it holds: + len(index_i) = len(index_j) = min(num_queries, num_target_boxes) + """ + + bs, num_queries = outputs["pred_logits"].shape[:2] + + # We flatten to compute the cost matrices in a batch + out_prob = outputs["pred_logits"].flatten(0, 1).sigmoid() # [batch_size * num_queries, num_classes] + out_bbox = outputs["pred_boxes"].flatten(0, 1) # [batch_size * num_queries, 4] + + # Also concat the target labels and boxes + tgt_ids = torch.cat([v["labels"] for v in targets]) + tgt_bbox = torch.cat([v["boxes"] for v in targets]) + + # Compute the classification cost. + alpha = self.focal_alpha + gamma = 2.0 + + new_label_map=label_map[tgt_ids.cpu()] + + neg_cost_class = (1 - alpha) * (out_prob ** gamma) * (-(1 - out_prob + 1e-8).log()) + pos_cost_class = alpha * ((1 - out_prob) ** gamma) * (-(out_prob + 1e-8).log()) + new_label_map=new_label_map.to(pos_cost_class.device) + cost_bbox = torch.cdist(out_bbox[:, :2], tgt_bbox[:, :2], p=1) + + # cost_class=(pos_cost_class @ new_label_map.T - neg_cost_class@ new_label_map.T) + cost_class=[] + for idx_map in new_label_map: + idx_map = idx_map / idx_map.sum() + cost_class.append(pos_cost_class @ idx_map - neg_cost_class@ idx_map) + if cost_class: + cost_class=torch.stack(cost_class,dim=0).T + else: + cost_class=torch.zeros_like(cost_bbox) + # Compute the L1 cost between boxes + + + # Compute the giou cost betwen boxes + cost_giou = -generalized_box_iou(box_cxcywh_to_xyxy(out_bbox), box_cxcywh_to_xyxy(tgt_bbox)) + # import pdb;pdb.set_trace() + # Final cost matrix + C = self.cost_bbox * cost_bbox + self.cost_class * cost_class + self.cost_giou * cost_giou + C = C.view(bs, num_queries, -1).cpu() + C[torch.isnan(C)] = 0.0 + C[torch.isinf(C)] = 0.0 + + sizes = [len(v["boxes"]) for v in targets] + try: + indices = [linear_sum_assignment(c[i]) for i, c in enumerate(C.split(sizes, -1))] + except: + print("warning: use SimpleMinsumMatcher") + indices = [] + device = C.device + for i, (c, _size) in enumerate(zip(C.split(sizes, -1), sizes)): + weight_mat = c[i] + idx_i = weight_mat.min(0)[1] + idx_j = torch.arange(_size).to(device) + indices.append((idx_i, idx_j)) + return [(torch.as_tensor(i, dtype=torch.int64), torch.as_tensor(j, dtype=torch.int64)) for i, j in indices] + + +class SimpleMinsumMatcher(nn.Module): + """This class computes an assignment between the targets and the predictions of the network + For efficiency reasons, the targets don't include the no_object. Because of this, in general, + there are more predictions than targets. In this case, we do a 1-to-1 matching of the best predictions, + while the others are un-matched (and thus treated as non-objects). + """ + + def __init__(self, cost_class: float = 1, cost_bbox: float = 1, cost_giou: float = 1, focal_alpha = 0.25): + """Creates the matcher + Params: + cost_class: This is the relative weight of the classification error in the matching cost + cost_bbox: This is the relative weight of the L1 error of the bounding box coordinates in the matching cost + cost_giou: This is the relative weight of the giou loss of the bounding box in the matching cost + """ + super().__init__() + self.cost_class = cost_class + self.cost_bbox = cost_bbox + self.cost_giou = cost_giou + assert cost_class != 0 or cost_bbox != 0 or cost_giou != 0, "all costs cant be 0" + + self.focal_alpha = focal_alpha + + @torch.no_grad() + def forward(self, outputs, targets): + """ Performs the matching + Params: + outputs: This is a dict that contains at least these entries: + "pred_logits": Tensor of dim [batch_size, num_queries, num_classes] with the classification logits + "pred_boxes": Tensor of dim [batch_size, num_queries, 4] with the predicted box coordinates + targets: This is a list of targets (len(targets) = batch_size), where each target is a dict containing: + "labels": Tensor of dim [num_target_boxes] (where num_target_boxes is the number of ground-truth + objects in the target) containing the class labels + "boxes": Tensor of dim [num_target_boxes, 4] containing the target box coordinates + Returns: + A list of size batch_size, containing tuples of (index_i, index_j) where: + - index_i is the indices of the selected predictions (in order) + - index_j is the indices of the corresponding selected targets (in order) + For each batch element, it holds: + len(index_i) = len(index_j) = min(num_queries, num_target_boxes) + """ + + bs, num_queries = outputs["pred_logits"].shape[:2] + + # We flatten to compute the cost matrices in a batch + out_prob = outputs["pred_logits"].flatten(0, 1).sigmoid() # [batch_size * num_queries, num_classes] + out_bbox = outputs["pred_boxes"].flatten(0, 1) # [batch_size * num_queries, 4] + + # Also concat the target labels and boxes + tgt_ids = torch.cat([v["labels"] for v in targets]) + tgt_bbox = torch.cat([v["boxes"] for v in targets]) + + # Compute the classification cost. + alpha = self.focal_alpha + gamma = 2.0 + neg_cost_class = (1 - alpha) * (out_prob ** gamma) * (-(1 - out_prob + 1e-8).log()) + pos_cost_class = alpha * ((1 - out_prob) ** gamma) * (-(out_prob + 1e-8).log()) + cost_class = pos_cost_class[:, tgt_ids] - neg_cost_class[:, tgt_ids] + + # Compute the L1 cost between boxes + cost_bbox = torch.cdist(out_bbox, tgt_bbox, p=1) + + # Compute the giou cost betwen boxes + cost_giou = -generalized_box_iou(box_cxcywh_to_xyxy(out_bbox), box_cxcywh_to_xyxy(tgt_bbox)) + + # Final cost matrix + + C = self.cost_bbox * cost_bbox + self.cost_class * cost_class + self.cost_giou * cost_giou + C = C.view(bs, num_queries, -1) + + sizes = [len(v["boxes"]) for v in targets] + indices = [] + device = C.device + for i, (c, _size) in enumerate(zip(C.split(sizes, -1), sizes)): + weight_mat = c[i] + idx_i = weight_mat.min(0)[1] + idx_j = torch.arange(_size).to(device) + indices.append((idx_i, idx_j)) + + return [(torch.as_tensor(i, dtype=torch.int64), torch.as_tensor(j, dtype=torch.int64)) for i, j in indices] + + +def build_matcher(args): + assert args.matcher_type in ['HungarianMatcher', 'SimpleMinsumMatcher'], "Unknown args.matcher_type: {}".format(args.matcher_type) + if args.matcher_type == 'HungarianMatcher': + return HungarianMatcher( + cost_class=args.set_cost_class, cost_bbox=args.set_cost_bbox, cost_giou=args.set_cost_giou, + focal_alpha=args.focal_alpha + ) + elif args.matcher_type == 'SimpleMinsumMatcher': + return SimpleMinsumMatcher( + cost_class=args.set_cost_class, cost_bbox=args.set_cost_bbox, cost_giou=args.set_cost_giou, + focal_alpha=args.focal_alpha + ) + else: + raise NotImplementedError("Unknown args.matcher_type: {}".format(args.matcher_type)) diff --git a/models/GroundingDINO/mlp_loca.py b/models/GroundingDINO/mlp_loca.py new file mode 100644 index 0000000000000000000000000000000000000000..7d39aedb5072ed9ca7d55774304e7f40aea9a60d --- /dev/null +++ b/models/GroundingDINO/mlp_loca.py @@ -0,0 +1,23 @@ +from torch import nn + + +class MLP(nn.Module): + + def __init__( + self, + input_dim: int, + hidden_dim: int, + dropout: float, + activation: nn.Module + ): + super(MLP, self).__init__() + + self.linear1 = nn.Linear(input_dim, hidden_dim) + self.linear2 = nn.Linear(hidden_dim, input_dim) + self.dropout = nn.Dropout(dropout) + self.activation = activation() + + def forward(self, x): + return ( + self.linear2(self.dropout(self.activation(self.linear1(x)))) + ) diff --git a/models/GroundingDINO/ms_deform_attn.py b/models/GroundingDINO/ms_deform_attn.py new file mode 100644 index 0000000000000000000000000000000000000000..9d0877d45e94656d3745bd82c40b9110e42dfc85 --- /dev/null +++ b/models/GroundingDINO/ms_deform_attn.py @@ -0,0 +1,414 @@ +# ------------------------------------------------------------------------ +# Grounding DINO +# url: https://github.com/IDEA-Research/GroundingDINO +# Copyright (c) 2023 IDEA. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------ +# Deformable DETR +# Copyright (c) 2020 SenseTime. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------------------------------ +# Modified from: +# https://github.com/fundamentalvision/Deformable-DETR/blob/main/models/ops/functions/ms_deform_attn_func.py +# https://github.com/fundamentalvision/Deformable-DETR/blob/main/models/ops/modules/ms_deform_attn.py +# https://github.com/open-mmlab/mmcv/blob/master/mmcv/ops/multi_scale_deform_attn.py +# ------------------------------------------------------------------------------------------------ + +import math +import warnings +from typing import Optional + +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.autograd import Function +from torch.autograd.function import once_differentiable +from torch.nn.init import constant_, xavier_uniform_ + +try: + # from groundingdino import _C + import MultiScaleDeformableAttention as _C +except: + warnings.warn("Failed to load custom C++ ops. Running on CPU mode Only!") + + +# helpers +def _is_power_of_2(n): + if (not isinstance(n, int)) or (n < 0): + raise ValueError("invalid input for _is_power_of_2: {} (type: {})".format(n, type(n))) + return (n & (n - 1) == 0) and n != 0 + + +class MultiScaleDeformableAttnFunction(Function): + @staticmethod + def forward( + ctx, + value, + value_spatial_shapes, + value_level_start_index, + sampling_locations, + attention_weights, + im2col_step, + ): + ctx.im2col_step = im2col_step + output = _C.ms_deform_attn_forward( + value, + value_spatial_shapes, + value_level_start_index, + sampling_locations, + attention_weights, + ctx.im2col_step, + ) + ctx.save_for_backward( + value, + value_spatial_shapes, + value_level_start_index, + sampling_locations, + attention_weights, + ) + return output + + @staticmethod + @once_differentiable + def backward(ctx, grad_output): + ( + value, + value_spatial_shapes, + value_level_start_index, + sampling_locations, + attention_weights, + ) = ctx.saved_tensors + grad_value, grad_sampling_loc, grad_attn_weight = _C.ms_deform_attn_backward( + value, + value_spatial_shapes, + value_level_start_index, + sampling_locations, + attention_weights, + grad_output, + ctx.im2col_step, + ) + + return grad_value, None, None, grad_sampling_loc, grad_attn_weight, None + + +def multi_scale_deformable_attn_pytorch( + value: torch.Tensor, + value_spatial_shapes: torch.Tensor, + sampling_locations: torch.Tensor, + attention_weights: torch.Tensor, +) -> torch.Tensor: + + bs, _, num_heads, embed_dims = value.shape + _, num_queries, num_heads, num_levels, num_points, _ = sampling_locations.shape + value_list = value.split([H_ * W_ for H_, W_ in value_spatial_shapes], dim=1) + sampling_grids = 2 * sampling_locations - 1 + sampling_value_list = [] + for level, (H_, W_) in enumerate(value_spatial_shapes): + # bs, H_*W_, num_heads, embed_dims -> + # bs, H_*W_, num_heads*embed_dims -> + # bs, num_heads*embed_dims, H_*W_ -> + # bs*num_heads, embed_dims, H_, W_ + value_l_ = ( + value_list[level].flatten(2).transpose(1, 2).reshape(bs * num_heads, embed_dims, H_, W_) + ) + # bs, num_queries, num_heads, num_points, 2 -> + # bs, num_heads, num_queries, num_points, 2 -> + # bs*num_heads, num_queries, num_points, 2 + sampling_grid_l_ = sampling_grids[:, :, :, level].transpose(1, 2).flatten(0, 1) + # bs*num_heads, embed_dims, num_queries, num_points + sampling_value_l_ = F.grid_sample( + value_l_, sampling_grid_l_, mode="bilinear", padding_mode="zeros", align_corners=False + ) + sampling_value_list.append(sampling_value_l_) + # (bs, num_queries, num_heads, num_levels, num_points) -> + # (bs, num_heads, num_queries, num_levels, num_points) -> + # (bs, num_heads, 1, num_queries, num_levels*num_points) + attention_weights = attention_weights.transpose(1, 2).reshape( + bs * num_heads, 1, num_queries, num_levels * num_points + ) + output = ( + (torch.stack(sampling_value_list, dim=-2).flatten(-2) * attention_weights) + .sum(-1) + .view(bs, num_heads * embed_dims, num_queries) + ) + return output.transpose(1, 2).contiguous() + + +class MultiScaleDeformableAttention(nn.Module): + """Multi-Scale Deformable Attention Module used in Deformable-DETR + + `Deformable DETR: Deformable Transformers for End-to-End Object Detection. + `_. + + Args: + embed_dim (int): The embedding dimension of Attention. Default: 256. + num_heads (int): The number of attention heads. Default: 8. + num_levels (int): The number of feature map used in Attention. Default: 4. + num_points (int): The number of sampling points for each query + in each head. Default: 4. + img2col_steps (int): The step used in image_to_column. Defualt: 64. + dropout (float): Dropout layer used in output. Default: 0.1. + batch_first (bool): if ``True``, then the input and output tensor will be + provided as `(bs, n, embed_dim)`. Default: False. `(n, bs, embed_dim)` + """ + + def __init__( + self, + embed_dim: int = 256, + num_heads: int = 8, + num_levels: int = 4, + num_points: int = 4, + img2col_step: int = 64, + batch_first: bool = False, + ): + super().__init__() + if embed_dim % num_heads != 0: + raise ValueError( + "embed_dim must be divisible by num_heads, but got {} and {}".format( + embed_dim, num_heads + ) + ) + head_dim = embed_dim // num_heads + + self.batch_first = batch_first + + if not _is_power_of_2(head_dim): + warnings.warn( + """ + You'd better set d_model in MSDeformAttn to make sure that + each dim of the attention head a power of 2, which is more efficient. + """ + ) + + self.im2col_step = img2col_step + self.embed_dim = embed_dim + self.num_heads = num_heads + self.num_levels = num_levels + self.num_points = num_points + self.sampling_offsets = nn.Linear(embed_dim, num_heads * num_levels * num_points * 2) + self.attention_weights = nn.Linear(embed_dim, num_heads * num_levels * num_points) + self.value_proj = nn.Linear(embed_dim, embed_dim) + self.output_proj = nn.Linear(embed_dim, embed_dim) + + self.init_weights() + + def _reset_parameters(self): + return self.init_weights() + + def init_weights(self): + """ + Default initialization for Parameters of Module. + """ + constant_(self.sampling_offsets.weight.data, 0.0) + thetas = torch.arange(self.num_heads, dtype=torch.float32) * ( + 2.0 * math.pi / self.num_heads + ) + grid_init = torch.stack([thetas.cos(), thetas.sin()], -1) + grid_init = ( + (grid_init / grid_init.abs().max(-1, keepdim=True)[0]) + .view(self.num_heads, 1, 1, 2) + .repeat(1, self.num_levels, self.num_points, 1) + ) + for i in range(self.num_points): + grid_init[:, :, i, :] *= i + 1 + with torch.no_grad(): + self.sampling_offsets.bias = nn.Parameter(grid_init.view(-1)) + constant_(self.attention_weights.weight.data, 0.0) + constant_(self.attention_weights.bias.data, 0.0) + xavier_uniform_(self.value_proj.weight.data) + constant_(self.value_proj.bias.data, 0.0) + xavier_uniform_(self.output_proj.weight.data) + constant_(self.output_proj.bias.data, 0.0) + + def freeze_sampling_offsets(self): + print("Freeze sampling offsets") + self.sampling_offsets.weight.requires_grad = False + self.sampling_offsets.bias.requires_grad = False + + def freeze_attention_weights(self): + print("Freeze attention weights") + self.attention_weights.weight.requires_grad = False + self.attention_weights.bias.requires_grad = False + + def forward( + self, + query: torch.Tensor, + key: Optional[torch.Tensor] = None, + value: Optional[torch.Tensor] = None, + query_pos: Optional[torch.Tensor] = None, + key_padding_mask: Optional[torch.Tensor] = None, + reference_points: Optional[torch.Tensor] = None, + spatial_shapes: Optional[torch.Tensor] = None, + level_start_index: Optional[torch.Tensor] = None, + **kwargs + ) -> torch.Tensor: + + """Forward Function of MultiScaleDeformableAttention + + Args: + query (torch.Tensor): Query embeddings with shape + `(num_query, bs, embed_dim)` + key (torch.Tensor): Key embeddings with shape + `(num_key, bs, embed_dim)` + value (torch.Tensor): Value embeddings with shape + `(num_key, bs, embed_dim)` + query_pos (torch.Tensor): The position embedding for `query`. Default: None. + key_padding_mask (torch.Tensor): ByteTensor for `query`, with shape `(bs, num_key)`, + indicating which elements within `key` to be ignored in attention. + reference_points (torch.Tensor): The normalized reference points + with shape `(bs, num_query, num_levels, 2)`, + all elements is range in [0, 1], top-left (0, 0), + bottom-right (1, 1), including padding are. + or `(N, Length_{query}, num_levels, 4)`, add additional + two dimensions `(h, w)` to form reference boxes. + spatial_shapes (torch.Tensor): Spatial shape of features in different levels. + With shape `(num_levels, 2)`, last dimension represents `(h, w)`. + level_start_index (torch.Tensor): The start index of each level. A tensor with + shape `(num_levels, )` which can be represented as + `[0, h_0 * w_0, h_0 * w_0 + h_1 * w_1, ...]`. + + Returns: + torch.Tensor: forward results with shape `(num_query, bs, embed_dim)` + """ + + if value is None: + value = query + + if query_pos is not None: + query = query + query_pos + + if not self.batch_first: + # change to (bs, num_query ,embed_dims) + query = query.permute(1, 0, 2) + value = value.permute(1, 0, 2) + + bs, num_query, _ = query.shape + bs, num_value, _ = value.shape + + assert (spatial_shapes[:, 0] * spatial_shapes[:, 1]).sum() == num_value + + value = self.value_proj(value) + if key_padding_mask is not None: + value = value.masked_fill(key_padding_mask[..., None], float(0)) + value = value.view(bs, num_value, self.num_heads, -1) + sampling_offsets = self.sampling_offsets(query).view( + bs, num_query, self.num_heads, self.num_levels, self.num_points, 2 + ) + attention_weights = self.attention_weights(query).view( + bs, num_query, self.num_heads, self.num_levels * self.num_points + ) + attention_weights = attention_weights.softmax(-1) + attention_weights = attention_weights.view( + bs, + num_query, + self.num_heads, + self.num_levels, + self.num_points, + ) + + # bs, num_query, num_heads, num_levels, num_points, 2 + if reference_points.shape[-1] == 2: + offset_normalizer = torch.stack([spatial_shapes[..., 1], spatial_shapes[..., 0]], -1) + sampling_locations = ( + reference_points[:, :, None, :, None, :] + + sampling_offsets / offset_normalizer[None, None, None, :, None, :] + ) + elif reference_points.shape[-1] == 4: + sampling_locations = ( + reference_points[:, :, None, :, None, :2] + + sampling_offsets + / self.num_points + * reference_points[:, :, None, :, None, 2:] + * 0.5 + ) + else: + raise ValueError( + "Last dim of reference_points must be 2 or 4, but get {} instead.".format( + reference_points.shape[-1] + ) + ) + + if torch.cuda.is_available() and value.is_cuda: + halffloat = False + if value.dtype == torch.float16: + halffloat = True + value = value.float() + sampling_locations = sampling_locations.float() + attention_weights = attention_weights.float() + + output = MultiScaleDeformableAttnFunction.apply( + value, + spatial_shapes, + level_start_index, + sampling_locations, + attention_weights, + self.im2col_step, + ) + + if halffloat: + output = output.half() + else: + output = multi_scale_deformable_attn_pytorch( + value, spatial_shapes, sampling_locations, attention_weights + ) + + output = self.output_proj(output) + + if not self.batch_first: + output = output.permute(1, 0, 2) + + return output + + +def create_dummy_class(klass, dependency, message=""): + """ + When a dependency of a class is not available, create a dummy class which throws ImportError + when used. + + Args: + klass (str): name of the class. + dependency (str): name of the dependency. + message: extra message to print + Returns: + class: a class object + """ + err = "Cannot import '{}', therefore '{}' is not available.".format(dependency, klass) + if message: + err = err + " " + message + + class _DummyMetaClass(type): + # throw error on class attribute access + def __getattr__(_, __): # noqa: B902 + raise ImportError(err) + + class _Dummy(object, metaclass=_DummyMetaClass): + # throw error on constructor + def __init__(self, *args, **kwargs): + raise ImportError(err) + + return _Dummy + + +def create_dummy_func(func, dependency, message=""): + """ + When a dependency of a function is not available, create a dummy function which throws + ImportError when used. + + Args: + func (str): name of the function. + dependency (str or list[str]): name(s) of the dependency. + message: extra message to print + Returns: + function: a function object + """ + err = "Cannot import '{}', therefore '{}' is not available.".format(dependency, func) + if message: + err = err + " " + message + + if isinstance(dependency, (list, tuple)): + dependency = ",".join(dependency) + + def _dummy(*args, **kwargs): + raise ImportError(err) + + return _dummy diff --git a/models/GroundingDINO/ops/build/lib.linux-x86_64-cpython-310/functions/__init__.py b/models/GroundingDINO/ops/build/lib.linux-x86_64-cpython-310/functions/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..8a2197bda3199aa32cafc5b9d396479609853dd2 --- /dev/null +++ b/models/GroundingDINO/ops/build/lib.linux-x86_64-cpython-310/functions/__init__.py @@ -0,0 +1,10 @@ +# ------------------------------------------------------------------------------------------------ +# Deformable DETR +# Copyright (c) 2020 SenseTime. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------------------------------ +# Modified from https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 +# ------------------------------------------------------------------------------------------------ + +from .ms_deform_attn_func import MSDeformAttnFunction + diff --git a/models/GroundingDINO/ops/build/lib.linux-x86_64-cpython-310/functions/ms_deform_attn_func.py b/models/GroundingDINO/ops/build/lib.linux-x86_64-cpython-310/functions/ms_deform_attn_func.py new file mode 100644 index 0000000000000000000000000000000000000000..8c5df8cf5d23aca963eec6c1133c180b37289607 --- /dev/null +++ b/models/GroundingDINO/ops/build/lib.linux-x86_64-cpython-310/functions/ms_deform_attn_func.py @@ -0,0 +1,61 @@ +# ------------------------------------------------------------------------------------------------ +# Deformable DETR +# Copyright (c) 2020 SenseTime. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------------------------------ +# Modified from https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 +# ------------------------------------------------------------------------------------------------ + +from __future__ import absolute_import +from __future__ import print_function +from __future__ import division + +import torch +import torch.nn.functional as F +from torch.autograd import Function +from torch.autograd.function import once_differentiable + +import MultiScaleDeformableAttention as MSDA + + +class MSDeformAttnFunction(Function): + @staticmethod + def forward(ctx, value, value_spatial_shapes, value_level_start_index, sampling_locations, attention_weights, im2col_step): + ctx.im2col_step = im2col_step + output = MSDA.ms_deform_attn_forward( + value, value_spatial_shapes, value_level_start_index, sampling_locations, attention_weights, ctx.im2col_step) + ctx.save_for_backward(value, value_spatial_shapes, value_level_start_index, sampling_locations, attention_weights) + return output + + @staticmethod + @once_differentiable + def backward(ctx, grad_output): + value, value_spatial_shapes, value_level_start_index, sampling_locations, attention_weights = ctx.saved_tensors + grad_value, grad_sampling_loc, grad_attn_weight = \ + MSDA.ms_deform_attn_backward( + value, value_spatial_shapes, value_level_start_index, sampling_locations, attention_weights, grad_output, ctx.im2col_step) + + return grad_value, None, None, grad_sampling_loc, grad_attn_weight, None + + +def ms_deform_attn_core_pytorch(value, value_spatial_shapes, sampling_locations, attention_weights): + # for debug and test only, + # need to use cuda version instead + N_, S_, M_, D_ = value.shape + _, Lq_, M_, L_, P_, _ = sampling_locations.shape + value_list = value.split([H_ * W_ for H_, W_ in value_spatial_shapes], dim=1) + sampling_grids = 2 * sampling_locations - 1 + sampling_value_list = [] + for lid_, (H_, W_) in enumerate(value_spatial_shapes): + # N_, H_*W_, M_, D_ -> N_, H_*W_, M_*D_ -> N_, M_*D_, H_*W_ -> N_*M_, D_, H_, W_ + value_l_ = value_list[lid_].flatten(2).transpose(1, 2).reshape(N_*M_, D_, H_, W_) + # N_, Lq_, M_, P_, 2 -> N_, M_, Lq_, P_, 2 -> N_*M_, Lq_, P_, 2 + sampling_grid_l_ = sampling_grids[:, :, :, lid_].transpose(1, 2).flatten(0, 1) + # N_*M_, D_, Lq_, P_ + sampling_value_l_ = F.grid_sample(value_l_, sampling_grid_l_, + mode='bilinear', padding_mode='zeros', align_corners=False) + sampling_value_list.append(sampling_value_l_) + # (N_, Lq_, M_, L_, P_) -> (N_, M_, Lq_, L_, P_) -> (N_, M_, 1, Lq_, L_*P_) + attention_weights = attention_weights.transpose(1, 2).reshape(N_*M_, 1, Lq_, L_*P_) + output = (torch.stack(sampling_value_list, dim=-2).flatten(-2) * attention_weights).sum(-1).view(N_, M_*D_, Lq_) + return output.transpose(1, 2).contiguous() diff --git a/models/GroundingDINO/ops/build/lib.linux-x86_64-cpython-310/modules/__init__.py b/models/GroundingDINO/ops/build/lib.linux-x86_64-cpython-310/modules/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..f82cb1ad9d634a87b54ba6a71b58a230bcade5fe --- /dev/null +++ b/models/GroundingDINO/ops/build/lib.linux-x86_64-cpython-310/modules/__init__.py @@ -0,0 +1,9 @@ +# ------------------------------------------------------------------------------------------------ +# Deformable DETR +# Copyright (c) 2020 SenseTime. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------------------------------ +# Modified from https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 +# ------------------------------------------------------------------------------------------------ + +from .ms_deform_attn import MSDeformAttn diff --git a/models/GroundingDINO/ops/build/lib.linux-x86_64-cpython-310/modules/ms_deform_attn.py b/models/GroundingDINO/ops/build/lib.linux-x86_64-cpython-310/modules/ms_deform_attn.py new file mode 100644 index 0000000000000000000000000000000000000000..0c22ac8b7611a7920115ce253bc9ef5510744dfd --- /dev/null +++ b/models/GroundingDINO/ops/build/lib.linux-x86_64-cpython-310/modules/ms_deform_attn.py @@ -0,0 +1,126 @@ +# ------------------------------------------------------------------------------------------------ +# Deformable DETR +# Copyright (c) 2020 SenseTime. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------------------------------ +# Modified from https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 +# ------------------------------------------------------------------------------------------------ + +from __future__ import absolute_import +from __future__ import print_function +from __future__ import division + +import warnings +import math + +import torch +from torch import nn +import torch.nn.functional as F +from torch.nn.init import xavier_uniform_, constant_ + +from ..functions import MSDeformAttnFunction + + +def _is_power_of_2(n): + if (not isinstance(n, int)) or (n < 0): + raise ValueError("invalid input for _is_power_of_2: {} (type: {})".format(n, type(n))) + return (n & (n-1) == 0) and n != 0 + + +class MSDeformAttn(nn.Module): + def __init__(self, d_model=256, n_levels=4, n_heads=8, n_points=4): + """ + Multi-Scale Deformable Attention Module + :param d_model hidden dimension + :param n_levels number of feature levels + :param n_heads number of attention heads + :param n_points number of sampling points per attention head per feature level + """ + super().__init__() + if d_model % n_heads != 0: + raise ValueError('d_model must be divisible by n_heads, but got {} and {}'.format(d_model, n_heads)) + _d_per_head = d_model // n_heads + # you'd better set _d_per_head to a power of 2 which is more efficient in our CUDA implementation + if not _is_power_of_2(_d_per_head): + warnings.warn("You'd better set d_model in MSDeformAttn to make the dimension of each attention head a power of 2 " + "which is more efficient in our CUDA implementation.") + + self.im2col_step = 64 + + self.d_model = d_model + self.n_levels = n_levels + self.n_heads = n_heads + self.n_points = n_points + + self.sampling_offsets = nn.Linear(d_model, n_heads * n_levels * n_points * 2) + self.attention_weights = nn.Linear(d_model, n_heads * n_levels * n_points) + self.value_proj = nn.Linear(d_model, d_model) + self.output_proj = nn.Linear(d_model, d_model) + + self._reset_parameters() + + def _reset_parameters(self): + constant_(self.sampling_offsets.weight.data, 0.) + thetas = torch.arange(self.n_heads, dtype=torch.float32) * (2.0 * math.pi / self.n_heads) + grid_init = torch.stack([thetas.cos(), thetas.sin()], -1) + grid_init = (grid_init / grid_init.abs().max(-1, keepdim=True)[0]).view(self.n_heads, 1, 1, 2).repeat(1, self.n_levels, self.n_points, 1) + for i in range(self.n_points): + grid_init[:, :, i, :] *= i + 1 + with torch.no_grad(): + self.sampling_offsets.bias = nn.Parameter(grid_init.view(-1)) + constant_(self.attention_weights.weight.data, 0.) + constant_(self.attention_weights.bias.data, 0.) + xavier_uniform_(self.value_proj.weight.data) + constant_(self.value_proj.bias.data, 0.) + xavier_uniform_(self.output_proj.weight.data) + constant_(self.output_proj.bias.data, 0.) + + def forward(self, query, reference_points, input_flatten, input_spatial_shapes, input_level_start_index, input_padding_mask=None): + """ + :param query (N, Length_{query}, C) + :param reference_points (N, Length_{query}, n_levels, 2), range in [0, 1], top-left (0,0), bottom-right (1, 1), including padding area + or (N, Length_{query}, n_levels, 4), add additional (w, h) to form reference boxes + :param input_flatten (N, \sum_{l=0}^{L-1} H_l \cdot W_l, C) + :param input_spatial_shapes (n_levels, 2), [(H_0, W_0), (H_1, W_1), ..., (H_{L-1}, W_{L-1})] + :param input_level_start_index (n_levels, ), [0, H_0*W_0, H_0*W_0+H_1*W_1, H_0*W_0+H_1*W_1+H_2*W_2, ..., H_0*W_0+H_1*W_1+...+H_{L-1}*W_{L-1}] + :param input_padding_mask (N, \sum_{l=0}^{L-1} H_l \cdot W_l), True for padding elements, False for non-padding elements + + :return output (N, Length_{query}, C) + """ + N, Len_q, _ = query.shape + N, Len_in, _ = input_flatten.shape + assert (input_spatial_shapes[:, 0] * input_spatial_shapes[:, 1]).sum() == Len_in + + value = self.value_proj(input_flatten) + if input_padding_mask is not None: + value = value.masked_fill(input_padding_mask[..., None], float(0)) + value = value.view(N, Len_in, self.n_heads, self.d_model // self.n_heads) + sampling_offsets = self.sampling_offsets(query).view(N, Len_q, self.n_heads, self.n_levels, self.n_points, 2) + attention_weights = self.attention_weights(query).view(N, Len_q, self.n_heads, self.n_levels * self.n_points) + attention_weights = F.softmax(attention_weights, -1).view(N, Len_q, self.n_heads, self.n_levels, self.n_points) + # N, Len_q, n_heads, n_levels, n_points, 2 + if reference_points.shape[-1] == 2: + offset_normalizer = torch.stack([input_spatial_shapes[..., 1], input_spatial_shapes[..., 0]], -1) + sampling_locations = reference_points[:, :, None, :, None, :] \ + + sampling_offsets / offset_normalizer[None, None, None, :, None, :] + elif reference_points.shape[-1] == 4: + sampling_locations = reference_points[:, :, None, :, None, :2] \ + + sampling_offsets / self.n_points * reference_points[:, :, None, :, None, 2:] * 0.5 + else: + raise ValueError( + 'Last dim of reference_points must be 2 or 4, but get {} instead.'.format(reference_points.shape[-1])) + + # for amp + if value.dtype == torch.float16: + # for mixed precision + output = MSDeformAttnFunction.apply( + value.to(torch.float32), input_spatial_shapes, input_level_start_index, sampling_locations.to(torch.float32), attention_weights, self.im2col_step) + output = output.to(torch.float16) + output = self.output_proj(output) + return output + + + output = MSDeformAttnFunction.apply( + value, input_spatial_shapes, input_level_start_index, sampling_locations, attention_weights, self.im2col_step) + output = self.output_proj(output) + return output diff --git a/models/GroundingDINO/ops/build/lib.linux-x86_64-cpython-39/functions/__init__.py b/models/GroundingDINO/ops/build/lib.linux-x86_64-cpython-39/functions/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..8a2197bda3199aa32cafc5b9d396479609853dd2 --- /dev/null +++ b/models/GroundingDINO/ops/build/lib.linux-x86_64-cpython-39/functions/__init__.py @@ -0,0 +1,10 @@ +# ------------------------------------------------------------------------------------------------ +# Deformable DETR +# Copyright (c) 2020 SenseTime. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------------------------------ +# Modified from https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 +# ------------------------------------------------------------------------------------------------ + +from .ms_deform_attn_func import MSDeformAttnFunction + diff --git a/models/GroundingDINO/ops/build/lib.linux-x86_64-cpython-39/functions/ms_deform_attn_func.py b/models/GroundingDINO/ops/build/lib.linux-x86_64-cpython-39/functions/ms_deform_attn_func.py new file mode 100644 index 0000000000000000000000000000000000000000..8c5df8cf5d23aca963eec6c1133c180b37289607 --- /dev/null +++ b/models/GroundingDINO/ops/build/lib.linux-x86_64-cpython-39/functions/ms_deform_attn_func.py @@ -0,0 +1,61 @@ +# ------------------------------------------------------------------------------------------------ +# Deformable DETR +# Copyright (c) 2020 SenseTime. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------------------------------ +# Modified from https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 +# ------------------------------------------------------------------------------------------------ + +from __future__ import absolute_import +from __future__ import print_function +from __future__ import division + +import torch +import torch.nn.functional as F +from torch.autograd import Function +from torch.autograd.function import once_differentiable + +import MultiScaleDeformableAttention as MSDA + + +class MSDeformAttnFunction(Function): + @staticmethod + def forward(ctx, value, value_spatial_shapes, value_level_start_index, sampling_locations, attention_weights, im2col_step): + ctx.im2col_step = im2col_step + output = MSDA.ms_deform_attn_forward( + value, value_spatial_shapes, value_level_start_index, sampling_locations, attention_weights, ctx.im2col_step) + ctx.save_for_backward(value, value_spatial_shapes, value_level_start_index, sampling_locations, attention_weights) + return output + + @staticmethod + @once_differentiable + def backward(ctx, grad_output): + value, value_spatial_shapes, value_level_start_index, sampling_locations, attention_weights = ctx.saved_tensors + grad_value, grad_sampling_loc, grad_attn_weight = \ + MSDA.ms_deform_attn_backward( + value, value_spatial_shapes, value_level_start_index, sampling_locations, attention_weights, grad_output, ctx.im2col_step) + + return grad_value, None, None, grad_sampling_loc, grad_attn_weight, None + + +def ms_deform_attn_core_pytorch(value, value_spatial_shapes, sampling_locations, attention_weights): + # for debug and test only, + # need to use cuda version instead + N_, S_, M_, D_ = value.shape + _, Lq_, M_, L_, P_, _ = sampling_locations.shape + value_list = value.split([H_ * W_ for H_, W_ in value_spatial_shapes], dim=1) + sampling_grids = 2 * sampling_locations - 1 + sampling_value_list = [] + for lid_, (H_, W_) in enumerate(value_spatial_shapes): + # N_, H_*W_, M_, D_ -> N_, H_*W_, M_*D_ -> N_, M_*D_, H_*W_ -> N_*M_, D_, H_, W_ + value_l_ = value_list[lid_].flatten(2).transpose(1, 2).reshape(N_*M_, D_, H_, W_) + # N_, Lq_, M_, P_, 2 -> N_, M_, Lq_, P_, 2 -> N_*M_, Lq_, P_, 2 + sampling_grid_l_ = sampling_grids[:, :, :, lid_].transpose(1, 2).flatten(0, 1) + # N_*M_, D_, Lq_, P_ + sampling_value_l_ = F.grid_sample(value_l_, sampling_grid_l_, + mode='bilinear', padding_mode='zeros', align_corners=False) + sampling_value_list.append(sampling_value_l_) + # (N_, Lq_, M_, L_, P_) -> (N_, M_, Lq_, L_, P_) -> (N_, M_, 1, Lq_, L_*P_) + attention_weights = attention_weights.transpose(1, 2).reshape(N_*M_, 1, Lq_, L_*P_) + output = (torch.stack(sampling_value_list, dim=-2).flatten(-2) * attention_weights).sum(-1).view(N_, M_*D_, Lq_) + return output.transpose(1, 2).contiguous() diff --git a/models/GroundingDINO/ops/build/lib.linux-x86_64-cpython-39/modules/__init__.py b/models/GroundingDINO/ops/build/lib.linux-x86_64-cpython-39/modules/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..f82cb1ad9d634a87b54ba6a71b58a230bcade5fe --- /dev/null +++ b/models/GroundingDINO/ops/build/lib.linux-x86_64-cpython-39/modules/__init__.py @@ -0,0 +1,9 @@ +# ------------------------------------------------------------------------------------------------ +# Deformable DETR +# Copyright (c) 2020 SenseTime. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------------------------------ +# Modified from https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 +# ------------------------------------------------------------------------------------------------ + +from .ms_deform_attn import MSDeformAttn diff --git a/models/GroundingDINO/ops/build/lib.linux-x86_64-cpython-39/modules/ms_deform_attn.py b/models/GroundingDINO/ops/build/lib.linux-x86_64-cpython-39/modules/ms_deform_attn.py new file mode 100644 index 0000000000000000000000000000000000000000..0c22ac8b7611a7920115ce253bc9ef5510744dfd --- /dev/null +++ b/models/GroundingDINO/ops/build/lib.linux-x86_64-cpython-39/modules/ms_deform_attn.py @@ -0,0 +1,126 @@ +# ------------------------------------------------------------------------------------------------ +# Deformable DETR +# Copyright (c) 2020 SenseTime. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------------------------------ +# Modified from https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 +# ------------------------------------------------------------------------------------------------ + +from __future__ import absolute_import +from __future__ import print_function +from __future__ import division + +import warnings +import math + +import torch +from torch import nn +import torch.nn.functional as F +from torch.nn.init import xavier_uniform_, constant_ + +from ..functions import MSDeformAttnFunction + + +def _is_power_of_2(n): + if (not isinstance(n, int)) or (n < 0): + raise ValueError("invalid input for _is_power_of_2: {} (type: {})".format(n, type(n))) + return (n & (n-1) == 0) and n != 0 + + +class MSDeformAttn(nn.Module): + def __init__(self, d_model=256, n_levels=4, n_heads=8, n_points=4): + """ + Multi-Scale Deformable Attention Module + :param d_model hidden dimension + :param n_levels number of feature levels + :param n_heads number of attention heads + :param n_points number of sampling points per attention head per feature level + """ + super().__init__() + if d_model % n_heads != 0: + raise ValueError('d_model must be divisible by n_heads, but got {} and {}'.format(d_model, n_heads)) + _d_per_head = d_model // n_heads + # you'd better set _d_per_head to a power of 2 which is more efficient in our CUDA implementation + if not _is_power_of_2(_d_per_head): + warnings.warn("You'd better set d_model in MSDeformAttn to make the dimension of each attention head a power of 2 " + "which is more efficient in our CUDA implementation.") + + self.im2col_step = 64 + + self.d_model = d_model + self.n_levels = n_levels + self.n_heads = n_heads + self.n_points = n_points + + self.sampling_offsets = nn.Linear(d_model, n_heads * n_levels * n_points * 2) + self.attention_weights = nn.Linear(d_model, n_heads * n_levels * n_points) + self.value_proj = nn.Linear(d_model, d_model) + self.output_proj = nn.Linear(d_model, d_model) + + self._reset_parameters() + + def _reset_parameters(self): + constant_(self.sampling_offsets.weight.data, 0.) + thetas = torch.arange(self.n_heads, dtype=torch.float32) * (2.0 * math.pi / self.n_heads) + grid_init = torch.stack([thetas.cos(), thetas.sin()], -1) + grid_init = (grid_init / grid_init.abs().max(-1, keepdim=True)[0]).view(self.n_heads, 1, 1, 2).repeat(1, self.n_levels, self.n_points, 1) + for i in range(self.n_points): + grid_init[:, :, i, :] *= i + 1 + with torch.no_grad(): + self.sampling_offsets.bias = nn.Parameter(grid_init.view(-1)) + constant_(self.attention_weights.weight.data, 0.) + constant_(self.attention_weights.bias.data, 0.) + xavier_uniform_(self.value_proj.weight.data) + constant_(self.value_proj.bias.data, 0.) + xavier_uniform_(self.output_proj.weight.data) + constant_(self.output_proj.bias.data, 0.) + + def forward(self, query, reference_points, input_flatten, input_spatial_shapes, input_level_start_index, input_padding_mask=None): + """ + :param query (N, Length_{query}, C) + :param reference_points (N, Length_{query}, n_levels, 2), range in [0, 1], top-left (0,0), bottom-right (1, 1), including padding area + or (N, Length_{query}, n_levels, 4), add additional (w, h) to form reference boxes + :param input_flatten (N, \sum_{l=0}^{L-1} H_l \cdot W_l, C) + :param input_spatial_shapes (n_levels, 2), [(H_0, W_0), (H_1, W_1), ..., (H_{L-1}, W_{L-1})] + :param input_level_start_index (n_levels, ), [0, H_0*W_0, H_0*W_0+H_1*W_1, H_0*W_0+H_1*W_1+H_2*W_2, ..., H_0*W_0+H_1*W_1+...+H_{L-1}*W_{L-1}] + :param input_padding_mask (N, \sum_{l=0}^{L-1} H_l \cdot W_l), True for padding elements, False for non-padding elements + + :return output (N, Length_{query}, C) + """ + N, Len_q, _ = query.shape + N, Len_in, _ = input_flatten.shape + assert (input_spatial_shapes[:, 0] * input_spatial_shapes[:, 1]).sum() == Len_in + + value = self.value_proj(input_flatten) + if input_padding_mask is not None: + value = value.masked_fill(input_padding_mask[..., None], float(0)) + value = value.view(N, Len_in, self.n_heads, self.d_model // self.n_heads) + sampling_offsets = self.sampling_offsets(query).view(N, Len_q, self.n_heads, self.n_levels, self.n_points, 2) + attention_weights = self.attention_weights(query).view(N, Len_q, self.n_heads, self.n_levels * self.n_points) + attention_weights = F.softmax(attention_weights, -1).view(N, Len_q, self.n_heads, self.n_levels, self.n_points) + # N, Len_q, n_heads, n_levels, n_points, 2 + if reference_points.shape[-1] == 2: + offset_normalizer = torch.stack([input_spatial_shapes[..., 1], input_spatial_shapes[..., 0]], -1) + sampling_locations = reference_points[:, :, None, :, None, :] \ + + sampling_offsets / offset_normalizer[None, None, None, :, None, :] + elif reference_points.shape[-1] == 4: + sampling_locations = reference_points[:, :, None, :, None, :2] \ + + sampling_offsets / self.n_points * reference_points[:, :, None, :, None, 2:] * 0.5 + else: + raise ValueError( + 'Last dim of reference_points must be 2 or 4, but get {} instead.'.format(reference_points.shape[-1])) + + # for amp + if value.dtype == torch.float16: + # for mixed precision + output = MSDeformAttnFunction.apply( + value.to(torch.float32), input_spatial_shapes, input_level_start_index, sampling_locations.to(torch.float32), attention_weights, self.im2col_step) + output = output.to(torch.float16) + output = self.output_proj(output) + return output + + + output = MSDeformAttnFunction.apply( + value, input_spatial_shapes, input_level_start_index, sampling_locations, attention_weights, self.im2col_step) + output = self.output_proj(output) + return output diff --git a/models/GroundingDINO/ops/functions/__init__.py b/models/GroundingDINO/ops/functions/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..8a2197bda3199aa32cafc5b9d396479609853dd2 --- /dev/null +++ b/models/GroundingDINO/ops/functions/__init__.py @@ -0,0 +1,10 @@ +# ------------------------------------------------------------------------------------------------ +# Deformable DETR +# Copyright (c) 2020 SenseTime. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------------------------------ +# Modified from https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 +# ------------------------------------------------------------------------------------------------ + +from .ms_deform_attn_func import MSDeformAttnFunction + diff --git a/models/GroundingDINO/ops/functions/ms_deform_attn_func.py b/models/GroundingDINO/ops/functions/ms_deform_attn_func.py new file mode 100644 index 0000000000000000000000000000000000000000..8c5df8cf5d23aca963eec6c1133c180b37289607 --- /dev/null +++ b/models/GroundingDINO/ops/functions/ms_deform_attn_func.py @@ -0,0 +1,61 @@ +# ------------------------------------------------------------------------------------------------ +# Deformable DETR +# Copyright (c) 2020 SenseTime. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------------------------------ +# Modified from https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 +# ------------------------------------------------------------------------------------------------ + +from __future__ import absolute_import +from __future__ import print_function +from __future__ import division + +import torch +import torch.nn.functional as F +from torch.autograd import Function +from torch.autograd.function import once_differentiable + +import MultiScaleDeformableAttention as MSDA + + +class MSDeformAttnFunction(Function): + @staticmethod + def forward(ctx, value, value_spatial_shapes, value_level_start_index, sampling_locations, attention_weights, im2col_step): + ctx.im2col_step = im2col_step + output = MSDA.ms_deform_attn_forward( + value, value_spatial_shapes, value_level_start_index, sampling_locations, attention_weights, ctx.im2col_step) + ctx.save_for_backward(value, value_spatial_shapes, value_level_start_index, sampling_locations, attention_weights) + return output + + @staticmethod + @once_differentiable + def backward(ctx, grad_output): + value, value_spatial_shapes, value_level_start_index, sampling_locations, attention_weights = ctx.saved_tensors + grad_value, grad_sampling_loc, grad_attn_weight = \ + MSDA.ms_deform_attn_backward( + value, value_spatial_shapes, value_level_start_index, sampling_locations, attention_weights, grad_output, ctx.im2col_step) + + return grad_value, None, None, grad_sampling_loc, grad_attn_weight, None + + +def ms_deform_attn_core_pytorch(value, value_spatial_shapes, sampling_locations, attention_weights): + # for debug and test only, + # need to use cuda version instead + N_, S_, M_, D_ = value.shape + _, Lq_, M_, L_, P_, _ = sampling_locations.shape + value_list = value.split([H_ * W_ for H_, W_ in value_spatial_shapes], dim=1) + sampling_grids = 2 * sampling_locations - 1 + sampling_value_list = [] + for lid_, (H_, W_) in enumerate(value_spatial_shapes): + # N_, H_*W_, M_, D_ -> N_, H_*W_, M_*D_ -> N_, M_*D_, H_*W_ -> N_*M_, D_, H_, W_ + value_l_ = value_list[lid_].flatten(2).transpose(1, 2).reshape(N_*M_, D_, H_, W_) + # N_, Lq_, M_, P_, 2 -> N_, M_, Lq_, P_, 2 -> N_*M_, Lq_, P_, 2 + sampling_grid_l_ = sampling_grids[:, :, :, lid_].transpose(1, 2).flatten(0, 1) + # N_*M_, D_, Lq_, P_ + sampling_value_l_ = F.grid_sample(value_l_, sampling_grid_l_, + mode='bilinear', padding_mode='zeros', align_corners=False) + sampling_value_list.append(sampling_value_l_) + # (N_, Lq_, M_, L_, P_) -> (N_, M_, Lq_, L_, P_) -> (N_, M_, 1, Lq_, L_*P_) + attention_weights = attention_weights.transpose(1, 2).reshape(N_*M_, 1, Lq_, L_*P_) + output = (torch.stack(sampling_value_list, dim=-2).flatten(-2) * attention_weights).sum(-1).view(N_, M_*D_, Lq_) + return output.transpose(1, 2).contiguous() diff --git a/models/GroundingDINO/ops/modules/__init__.py b/models/GroundingDINO/ops/modules/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..f82cb1ad9d634a87b54ba6a71b58a230bcade5fe --- /dev/null +++ b/models/GroundingDINO/ops/modules/__init__.py @@ -0,0 +1,9 @@ +# ------------------------------------------------------------------------------------------------ +# Deformable DETR +# Copyright (c) 2020 SenseTime. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------------------------------ +# Modified from https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 +# ------------------------------------------------------------------------------------------------ + +from .ms_deform_attn import MSDeformAttn diff --git a/models/GroundingDINO/ops/modules/ms_deform_attn.py b/models/GroundingDINO/ops/modules/ms_deform_attn.py new file mode 100644 index 0000000000000000000000000000000000000000..0c22ac8b7611a7920115ce253bc9ef5510744dfd --- /dev/null +++ b/models/GroundingDINO/ops/modules/ms_deform_attn.py @@ -0,0 +1,126 @@ +# ------------------------------------------------------------------------------------------------ +# Deformable DETR +# Copyright (c) 2020 SenseTime. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------------------------------ +# Modified from https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 +# ------------------------------------------------------------------------------------------------ + +from __future__ import absolute_import +from __future__ import print_function +from __future__ import division + +import warnings +import math + +import torch +from torch import nn +import torch.nn.functional as F +from torch.nn.init import xavier_uniform_, constant_ + +from ..functions import MSDeformAttnFunction + + +def _is_power_of_2(n): + if (not isinstance(n, int)) or (n < 0): + raise ValueError("invalid input for _is_power_of_2: {} (type: {})".format(n, type(n))) + return (n & (n-1) == 0) and n != 0 + + +class MSDeformAttn(nn.Module): + def __init__(self, d_model=256, n_levels=4, n_heads=8, n_points=4): + """ + Multi-Scale Deformable Attention Module + :param d_model hidden dimension + :param n_levels number of feature levels + :param n_heads number of attention heads + :param n_points number of sampling points per attention head per feature level + """ + super().__init__() + if d_model % n_heads != 0: + raise ValueError('d_model must be divisible by n_heads, but got {} and {}'.format(d_model, n_heads)) + _d_per_head = d_model // n_heads + # you'd better set _d_per_head to a power of 2 which is more efficient in our CUDA implementation + if not _is_power_of_2(_d_per_head): + warnings.warn("You'd better set d_model in MSDeformAttn to make the dimension of each attention head a power of 2 " + "which is more efficient in our CUDA implementation.") + + self.im2col_step = 64 + + self.d_model = d_model + self.n_levels = n_levels + self.n_heads = n_heads + self.n_points = n_points + + self.sampling_offsets = nn.Linear(d_model, n_heads * n_levels * n_points * 2) + self.attention_weights = nn.Linear(d_model, n_heads * n_levels * n_points) + self.value_proj = nn.Linear(d_model, d_model) + self.output_proj = nn.Linear(d_model, d_model) + + self._reset_parameters() + + def _reset_parameters(self): + constant_(self.sampling_offsets.weight.data, 0.) + thetas = torch.arange(self.n_heads, dtype=torch.float32) * (2.0 * math.pi / self.n_heads) + grid_init = torch.stack([thetas.cos(), thetas.sin()], -1) + grid_init = (grid_init / grid_init.abs().max(-1, keepdim=True)[0]).view(self.n_heads, 1, 1, 2).repeat(1, self.n_levels, self.n_points, 1) + for i in range(self.n_points): + grid_init[:, :, i, :] *= i + 1 + with torch.no_grad(): + self.sampling_offsets.bias = nn.Parameter(grid_init.view(-1)) + constant_(self.attention_weights.weight.data, 0.) + constant_(self.attention_weights.bias.data, 0.) + xavier_uniform_(self.value_proj.weight.data) + constant_(self.value_proj.bias.data, 0.) + xavier_uniform_(self.output_proj.weight.data) + constant_(self.output_proj.bias.data, 0.) + + def forward(self, query, reference_points, input_flatten, input_spatial_shapes, input_level_start_index, input_padding_mask=None): + """ + :param query (N, Length_{query}, C) + :param reference_points (N, Length_{query}, n_levels, 2), range in [0, 1], top-left (0,0), bottom-right (1, 1), including padding area + or (N, Length_{query}, n_levels, 4), add additional (w, h) to form reference boxes + :param input_flatten (N, \sum_{l=0}^{L-1} H_l \cdot W_l, C) + :param input_spatial_shapes (n_levels, 2), [(H_0, W_0), (H_1, W_1), ..., (H_{L-1}, W_{L-1})] + :param input_level_start_index (n_levels, ), [0, H_0*W_0, H_0*W_0+H_1*W_1, H_0*W_0+H_1*W_1+H_2*W_2, ..., H_0*W_0+H_1*W_1+...+H_{L-1}*W_{L-1}] + :param input_padding_mask (N, \sum_{l=0}^{L-1} H_l \cdot W_l), True for padding elements, False for non-padding elements + + :return output (N, Length_{query}, C) + """ + N, Len_q, _ = query.shape + N, Len_in, _ = input_flatten.shape + assert (input_spatial_shapes[:, 0] * input_spatial_shapes[:, 1]).sum() == Len_in + + value = self.value_proj(input_flatten) + if input_padding_mask is not None: + value = value.masked_fill(input_padding_mask[..., None], float(0)) + value = value.view(N, Len_in, self.n_heads, self.d_model // self.n_heads) + sampling_offsets = self.sampling_offsets(query).view(N, Len_q, self.n_heads, self.n_levels, self.n_points, 2) + attention_weights = self.attention_weights(query).view(N, Len_q, self.n_heads, self.n_levels * self.n_points) + attention_weights = F.softmax(attention_weights, -1).view(N, Len_q, self.n_heads, self.n_levels, self.n_points) + # N, Len_q, n_heads, n_levels, n_points, 2 + if reference_points.shape[-1] == 2: + offset_normalizer = torch.stack([input_spatial_shapes[..., 1], input_spatial_shapes[..., 0]], -1) + sampling_locations = reference_points[:, :, None, :, None, :] \ + + sampling_offsets / offset_normalizer[None, None, None, :, None, :] + elif reference_points.shape[-1] == 4: + sampling_locations = reference_points[:, :, None, :, None, :2] \ + + sampling_offsets / self.n_points * reference_points[:, :, None, :, None, 2:] * 0.5 + else: + raise ValueError( + 'Last dim of reference_points must be 2 or 4, but get {} instead.'.format(reference_points.shape[-1])) + + # for amp + if value.dtype == torch.float16: + # for mixed precision + output = MSDeformAttnFunction.apply( + value.to(torch.float32), input_spatial_shapes, input_level_start_index, sampling_locations.to(torch.float32), attention_weights, self.im2col_step) + output = output.to(torch.float16) + output = self.output_proj(output) + return output + + + output = MSDeformAttnFunction.apply( + value, input_spatial_shapes, input_level_start_index, sampling_locations, attention_weights, self.im2col_step) + output = self.output_proj(output) + return output diff --git a/models/GroundingDINO/ops/setup.py b/models/GroundingDINO/ops/setup.py new file mode 100644 index 0000000000000000000000000000000000000000..c76d823200c0a3bbad37444958e8af8889fe653e --- /dev/null +++ b/models/GroundingDINO/ops/setup.py @@ -0,0 +1,73 @@ +# ------------------------------------------------------------------------------------------------ +# Deformable DETR +# Copyright (c) 2020 SenseTime. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------------------------------ +# Modified from https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 +# ------------------------------------------------------------------------------------------------ + +import os +import glob + +import torch + +from torch.utils.cpp_extension import CUDA_HOME +from torch.utils.cpp_extension import CppExtension +from torch.utils.cpp_extension import CUDAExtension + +from setuptools import find_packages +from setuptools import setup + +requirements = ["torch", "torchvision"] + +def get_extensions(): + this_dir = os.path.dirname(os.path.abspath(__file__)) + extensions_dir = os.path.join(this_dir, "src") + + main_file = glob.glob(os.path.join(extensions_dir, "*.cpp")) + source_cpu = glob.glob(os.path.join(extensions_dir, "cpu", "*.cpp")) + source_cuda = glob.glob(os.path.join(extensions_dir, "cuda", "*.cu")) + + sources = main_file + source_cpu + extension = CppExtension + extra_compile_args = {"cxx": []} + define_macros = [] + + + + if torch.cuda.is_available() and CUDA_HOME is not None: + extension = CUDAExtension + sources += source_cuda + define_macros += [("WITH_CUDA", None)] + extra_compile_args["nvcc"] = [ + "-DCUDA_HAS_FP16=1", + "-D__CUDA_NO_HALF_OPERATORS__", + "-D__CUDA_NO_HALF_CONVERSIONS__", + "-D__CUDA_NO_HALF2_OPERATORS__", + ] + else: + raise NotImplementedError('Cuda is not availabel') + + sources = [os.path.join(extensions_dir, s) for s in sources] + include_dirs = [extensions_dir] + ext_modules = [ + extension( + "MultiScaleDeformableAttention", + sources, + include_dirs=include_dirs, + define_macros=define_macros, + extra_compile_args=extra_compile_args, + ) + ] + return ext_modules + +setup( + name="MultiScaleDeformableAttention", + version="1.0", + author="Weijie Su", + url="https://github.com/fundamentalvision/Deformable-DETR", + description="PyTorch Wrapper for CUDA Functions of Multi-Scale Deformable Attention", + packages=find_packages(exclude=("configs", "tests",)), + ext_modules=get_extensions(), + cmdclass={"build_ext": torch.utils.cpp_extension.BuildExtension}, +) diff --git a/models/GroundingDINO/ops/src/cpu/ms_deform_attn_cpu.cpp b/models/GroundingDINO/ops/src/cpu/ms_deform_attn_cpu.cpp new file mode 100644 index 0000000000000000000000000000000000000000..e1bf854de1f3860d20b6fef5c1a17817c268e70a --- /dev/null +++ b/models/GroundingDINO/ops/src/cpu/ms_deform_attn_cpu.cpp @@ -0,0 +1,41 @@ +/*! +************************************************************************************************** +* Deformable DETR +* Copyright (c) 2020 SenseTime. All Rights Reserved. +* Licensed under the Apache License, Version 2.0 [see LICENSE for details] +************************************************************************************************** +* Modified from https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 +************************************************************************************************** +*/ + +#include + +#include +#include + + +at::Tensor +ms_deform_attn_cpu_forward( + const at::Tensor &value, + const at::Tensor &spatial_shapes, + const at::Tensor &level_start_index, + const at::Tensor &sampling_loc, + const at::Tensor &attn_weight, + const int im2col_step) +{ + AT_ERROR("Not implement on cpu"); +} + +std::vector +ms_deform_attn_cpu_backward( + const at::Tensor &value, + const at::Tensor &spatial_shapes, + const at::Tensor &level_start_index, + const at::Tensor &sampling_loc, + const at::Tensor &attn_weight, + const at::Tensor &grad_output, + const int im2col_step) +{ + AT_ERROR("Not implement on cpu"); +} + diff --git a/models/GroundingDINO/ops/src/cpu/ms_deform_attn_cpu.h b/models/GroundingDINO/ops/src/cpu/ms_deform_attn_cpu.h new file mode 100644 index 0000000000000000000000000000000000000000..81b7b58a3d9502bbb684dc84687a526dedf94cae --- /dev/null +++ b/models/GroundingDINO/ops/src/cpu/ms_deform_attn_cpu.h @@ -0,0 +1,33 @@ +/*! +************************************************************************************************** +* Deformable DETR +* Copyright (c) 2020 SenseTime. All Rights Reserved. +* Licensed under the Apache License, Version 2.0 [see LICENSE for details] +************************************************************************************************** +* Modified from https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 +************************************************************************************************** +*/ + +#pragma once +#include + +at::Tensor +ms_deform_attn_cpu_forward( + const at::Tensor &value, + const at::Tensor &spatial_shapes, + const at::Tensor &level_start_index, + const at::Tensor &sampling_loc, + const at::Tensor &attn_weight, + const int im2col_step); + +std::vector +ms_deform_attn_cpu_backward( + const at::Tensor &value, + const at::Tensor &spatial_shapes, + const at::Tensor &level_start_index, + const at::Tensor &sampling_loc, + const at::Tensor &attn_weight, + const at::Tensor &grad_output, + const int im2col_step); + + diff --git a/models/GroundingDINO/ops/src/cuda/ms_deform_attn_cuda.cu b/models/GroundingDINO/ops/src/cuda/ms_deform_attn_cuda.cu new file mode 100644 index 0000000000000000000000000000000000000000..d6d583647cce987196d5ad1968a8a365a379e774 --- /dev/null +++ b/models/GroundingDINO/ops/src/cuda/ms_deform_attn_cuda.cu @@ -0,0 +1,153 @@ +/*! +************************************************************************************************** +* Deformable DETR +* Copyright (c) 2020 SenseTime. All Rights Reserved. +* Licensed under the Apache License, Version 2.0 [see LICENSE for details] +************************************************************************************************** +* Modified from https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 +************************************************************************************************** +*/ + +#include +#include "cuda/ms_deform_im2col_cuda.cuh" + +#include +#include +#include +#include + + +at::Tensor ms_deform_attn_cuda_forward( + const at::Tensor &value, + const at::Tensor &spatial_shapes, + const at::Tensor &level_start_index, + const at::Tensor &sampling_loc, + const at::Tensor &attn_weight, + const int im2col_step) +{ + AT_ASSERTM(value.is_contiguous(), "value tensor has to be contiguous"); + AT_ASSERTM(spatial_shapes.is_contiguous(), "spatial_shapes tensor has to be contiguous"); + AT_ASSERTM(level_start_index.is_contiguous(), "level_start_index tensor has to be contiguous"); + AT_ASSERTM(sampling_loc.is_contiguous(), "sampling_loc tensor has to be contiguous"); + AT_ASSERTM(attn_weight.is_contiguous(), "attn_weight tensor has to be contiguous"); + + AT_ASSERTM(value.type().is_cuda(), "value must be a CUDA tensor"); + AT_ASSERTM(spatial_shapes.type().is_cuda(), "spatial_shapes must be a CUDA tensor"); + AT_ASSERTM(level_start_index.type().is_cuda(), "level_start_index must be a CUDA tensor"); + AT_ASSERTM(sampling_loc.type().is_cuda(), "sampling_loc must be a CUDA tensor"); + AT_ASSERTM(attn_weight.type().is_cuda(), "attn_weight must be a CUDA tensor"); + + const int batch = value.size(0); + const int spatial_size = value.size(1); + const int num_heads = value.size(2); + const int channels = value.size(3); + + const int num_levels = spatial_shapes.size(0); + + const int num_query = sampling_loc.size(1); + const int num_point = sampling_loc.size(4); + + const int im2col_step_ = std::min(batch, im2col_step); + + AT_ASSERTM(batch % im2col_step_ == 0, "batch(%d) must divide im2col_step(%d)", batch, im2col_step_); + + auto output = at::zeros({batch, num_query, num_heads, channels}, value.options()); + + const int batch_n = im2col_step_; + auto output_n = output.view({batch/im2col_step_, batch_n, num_query, num_heads, channels}); + auto per_value_size = spatial_size * num_heads * channels; + auto per_sample_loc_size = num_query * num_heads * num_levels * num_point * 2; + auto per_attn_weight_size = num_query * num_heads * num_levels * num_point; + for (int n = 0; n < batch/im2col_step_; ++n) + { + auto columns = output_n.select(0, n); + AT_DISPATCH_FLOATING_TYPES(value.type(), "ms_deform_attn_forward_cuda", ([&] { + ms_deformable_im2col_cuda(at::cuda::getCurrentCUDAStream(), + value.data() + n * im2col_step_ * per_value_size, + spatial_shapes.data(), + level_start_index.data(), + sampling_loc.data() + n * im2col_step_ * per_sample_loc_size, + attn_weight.data() + n * im2col_step_ * per_attn_weight_size, + batch_n, spatial_size, num_heads, channels, num_levels, num_query, num_point, + columns.data()); + + })); + } + + output = output.view({batch, num_query, num_heads*channels}); + + return output; +} + + +std::vector ms_deform_attn_cuda_backward( + const at::Tensor &value, + const at::Tensor &spatial_shapes, + const at::Tensor &level_start_index, + const at::Tensor &sampling_loc, + const at::Tensor &attn_weight, + const at::Tensor &grad_output, + const int im2col_step) +{ + + AT_ASSERTM(value.is_contiguous(), "value tensor has to be contiguous"); + AT_ASSERTM(spatial_shapes.is_contiguous(), "spatial_shapes tensor has to be contiguous"); + AT_ASSERTM(level_start_index.is_contiguous(), "level_start_index tensor has to be contiguous"); + AT_ASSERTM(sampling_loc.is_contiguous(), "sampling_loc tensor has to be contiguous"); + AT_ASSERTM(attn_weight.is_contiguous(), "attn_weight tensor has to be contiguous"); + AT_ASSERTM(grad_output.is_contiguous(), "grad_output tensor has to be contiguous"); + + AT_ASSERTM(value.type().is_cuda(), "value must be a CUDA tensor"); + AT_ASSERTM(spatial_shapes.type().is_cuda(), "spatial_shapes must be a CUDA tensor"); + AT_ASSERTM(level_start_index.type().is_cuda(), "level_start_index must be a CUDA tensor"); + AT_ASSERTM(sampling_loc.type().is_cuda(), "sampling_loc must be a CUDA tensor"); + AT_ASSERTM(attn_weight.type().is_cuda(), "attn_weight must be a CUDA tensor"); + AT_ASSERTM(grad_output.type().is_cuda(), "grad_output must be a CUDA tensor"); + + const int batch = value.size(0); + const int spatial_size = value.size(1); + const int num_heads = value.size(2); + const int channels = value.size(3); + + const int num_levels = spatial_shapes.size(0); + + const int num_query = sampling_loc.size(1); + const int num_point = sampling_loc.size(4); + + const int im2col_step_ = std::min(batch, im2col_step); + + AT_ASSERTM(batch % im2col_step_ == 0, "batch(%d) must divide im2col_step(%d)", batch, im2col_step_); + + auto grad_value = at::zeros_like(value); + auto grad_sampling_loc = at::zeros_like(sampling_loc); + auto grad_attn_weight = at::zeros_like(attn_weight); + + const int batch_n = im2col_step_; + auto per_value_size = spatial_size * num_heads * channels; + auto per_sample_loc_size = num_query * num_heads * num_levels * num_point * 2; + auto per_attn_weight_size = num_query * num_heads * num_levels * num_point; + auto grad_output_n = grad_output.view({batch/im2col_step_, batch_n, num_query, num_heads, channels}); + + for (int n = 0; n < batch/im2col_step_; ++n) + { + auto grad_output_g = grad_output_n.select(0, n); + AT_DISPATCH_FLOATING_TYPES(value.type(), "ms_deform_attn_backward_cuda", ([&] { + ms_deformable_col2im_cuda(at::cuda::getCurrentCUDAStream(), + grad_output_g.data(), + value.data() + n * im2col_step_ * per_value_size, + spatial_shapes.data(), + level_start_index.data(), + sampling_loc.data() + n * im2col_step_ * per_sample_loc_size, + attn_weight.data() + n * im2col_step_ * per_attn_weight_size, + batch_n, spatial_size, num_heads, channels, num_levels, num_query, num_point, + grad_value.data() + n * im2col_step_ * per_value_size, + grad_sampling_loc.data() + n * im2col_step_ * per_sample_loc_size, + grad_attn_weight.data() + n * im2col_step_ * per_attn_weight_size); + + })); + } + + return { + grad_value, grad_sampling_loc, grad_attn_weight + }; +} \ No newline at end of file diff --git a/models/GroundingDINO/ops/src/cuda/ms_deform_attn_cuda.h b/models/GroundingDINO/ops/src/cuda/ms_deform_attn_cuda.h new file mode 100644 index 0000000000000000000000000000000000000000..c7ae53f99c820ce6193b608ad344550348a0b42c --- /dev/null +++ b/models/GroundingDINO/ops/src/cuda/ms_deform_attn_cuda.h @@ -0,0 +1,30 @@ +/*! +************************************************************************************************** +* Deformable DETR +* Copyright (c) 2020 SenseTime. All Rights Reserved. +* Licensed under the Apache License, Version 2.0 [see LICENSE for details] +************************************************************************************************** +* Modified from https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 +************************************************************************************************** +*/ + +#pragma once +#include + +at::Tensor ms_deform_attn_cuda_forward( + const at::Tensor &value, + const at::Tensor &spatial_shapes, + const at::Tensor &level_start_index, + const at::Tensor &sampling_loc, + const at::Tensor &attn_weight, + const int im2col_step); + +std::vector ms_deform_attn_cuda_backward( + const at::Tensor &value, + const at::Tensor &spatial_shapes, + const at::Tensor &level_start_index, + const at::Tensor &sampling_loc, + const at::Tensor &attn_weight, + const at::Tensor &grad_output, + const int im2col_step); + diff --git a/models/GroundingDINO/ops/src/cuda/ms_deform_im2col_cuda.cuh b/models/GroundingDINO/ops/src/cuda/ms_deform_im2col_cuda.cuh new file mode 100644 index 0000000000000000000000000000000000000000..6bc2acb7aea0eab2e9e91e769a16861e1652c284 --- /dev/null +++ b/models/GroundingDINO/ops/src/cuda/ms_deform_im2col_cuda.cuh @@ -0,0 +1,1327 @@ +/*! +************************************************************************** +* Deformable DETR +* Copyright (c) 2020 SenseTime. All Rights Reserved. +* Licensed under the Apache License, Version 2.0 [see LICENSE for details] +************************************************************************** +* Modified from DCN (https://github.com/msracver/Deformable-ConvNets) +* Copyright (c) 2018 Microsoft +************************************************************************** +*/ + +#include +#include +#include + +#include +#include + +#include + +#define CUDA_KERNEL_LOOP(i, n) \ + for (int i = blockIdx.x * blockDim.x + threadIdx.x; \ + i < (n); \ + i += blockDim.x * gridDim.x) + +const int CUDA_NUM_THREADS = 1024; +inline int GET_BLOCKS(const int N, const int num_threads) +{ + return (N + num_threads - 1) / num_threads; +} + + +template +__device__ scalar_t ms_deform_attn_im2col_bilinear(const scalar_t* &bottom_data, + const int &height, const int &width, const int &nheads, const int &channels, + const scalar_t &h, const scalar_t &w, const int &m, const int &c) +{ + const int h_low = floor(h); + const int w_low = floor(w); + const int h_high = h_low + 1; + const int w_high = w_low + 1; + + const scalar_t lh = h - h_low; + const scalar_t lw = w - w_low; + const scalar_t hh = 1 - lh, hw = 1 - lw; + + const int w_stride = nheads * channels; + const int h_stride = width * w_stride; + const int h_low_ptr_offset = h_low * h_stride; + const int h_high_ptr_offset = h_low_ptr_offset + h_stride; + const int w_low_ptr_offset = w_low * w_stride; + const int w_high_ptr_offset = w_low_ptr_offset + w_stride; + const int base_ptr = m * channels + c; + + scalar_t v1 = 0; + if (h_low >= 0 && w_low >= 0) + { + const int ptr1 = h_low_ptr_offset + w_low_ptr_offset + base_ptr; + v1 = bottom_data[ptr1]; + } + scalar_t v2 = 0; + if (h_low >= 0 && w_high <= width - 1) + { + const int ptr2 = h_low_ptr_offset + w_high_ptr_offset + base_ptr; + v2 = bottom_data[ptr2]; + } + scalar_t v3 = 0; + if (h_high <= height - 1 && w_low >= 0) + { + const int ptr3 = h_high_ptr_offset + w_low_ptr_offset + base_ptr; + v3 = bottom_data[ptr3]; + } + scalar_t v4 = 0; + if (h_high <= height - 1 && w_high <= width - 1) + { + const int ptr4 = h_high_ptr_offset + w_high_ptr_offset + base_ptr; + v4 = bottom_data[ptr4]; + } + + const scalar_t w1 = hh * hw, w2 = hh * lw, w3 = lh * hw, w4 = lh * lw; + + const scalar_t val = (w1 * v1 + w2 * v2 + w3 * v3 + w4 * v4); + return val; +} + + +template +__device__ void ms_deform_attn_col2im_bilinear(const scalar_t* &bottom_data, + const int &height, const int &width, const int &nheads, const int &channels, + const scalar_t &h, const scalar_t &w, const int &m, const int &c, + const scalar_t &top_grad, + const scalar_t &attn_weight, + scalar_t* &grad_value, + scalar_t* grad_sampling_loc, + scalar_t* grad_attn_weight) +{ + const int h_low = floor(h); + const int w_low = floor(w); + const int h_high = h_low + 1; + const int w_high = w_low + 1; + + const scalar_t lh = h - h_low; + const scalar_t lw = w - w_low; + const scalar_t hh = 1 - lh, hw = 1 - lw; + + const int w_stride = nheads * channels; + const int h_stride = width * w_stride; + const int h_low_ptr_offset = h_low * h_stride; + const int h_high_ptr_offset = h_low_ptr_offset + h_stride; + const int w_low_ptr_offset = w_low * w_stride; + const int w_high_ptr_offset = w_low_ptr_offset + w_stride; + const int base_ptr = m * channels + c; + + const scalar_t w1 = hh * hw, w2 = hh * lw, w3 = lh * hw, w4 = lh * lw; + const scalar_t top_grad_value = top_grad * attn_weight; + scalar_t grad_h_weight = 0, grad_w_weight = 0; + + scalar_t v1 = 0; + if (h_low >= 0 && w_low >= 0) + { + const int ptr1 = h_low_ptr_offset + w_low_ptr_offset + base_ptr; + v1 = bottom_data[ptr1]; + grad_h_weight -= hw * v1; + grad_w_weight -= hh * v1; + atomicAdd(grad_value+ptr1, w1*top_grad_value); + } + scalar_t v2 = 0; + if (h_low >= 0 && w_high <= width - 1) + { + const int ptr2 = h_low_ptr_offset + w_high_ptr_offset + base_ptr; + v2 = bottom_data[ptr2]; + grad_h_weight -= lw * v2; + grad_w_weight += hh * v2; + atomicAdd(grad_value+ptr2, w2*top_grad_value); + } + scalar_t v3 = 0; + if (h_high <= height - 1 && w_low >= 0) + { + const int ptr3 = h_high_ptr_offset + w_low_ptr_offset + base_ptr; + v3 = bottom_data[ptr3]; + grad_h_weight += hw * v3; + grad_w_weight -= lh * v3; + atomicAdd(grad_value+ptr3, w3*top_grad_value); + } + scalar_t v4 = 0; + if (h_high <= height - 1 && w_high <= width - 1) + { + const int ptr4 = h_high_ptr_offset + w_high_ptr_offset + base_ptr; + v4 = bottom_data[ptr4]; + grad_h_weight += lw * v4; + grad_w_weight += lh * v4; + atomicAdd(grad_value+ptr4, w4*top_grad_value); + } + + const scalar_t val = (w1 * v1 + w2 * v2 + w3 * v3 + w4 * v4); + *grad_attn_weight = top_grad * val; + *grad_sampling_loc = width * grad_w_weight * top_grad_value; + *(grad_sampling_loc + 1) = height * grad_h_weight * top_grad_value; +} + + +template +__device__ void ms_deform_attn_col2im_bilinear_gm(const scalar_t* &bottom_data, + const int &height, const int &width, const int &nheads, const int &channels, + const scalar_t &h, const scalar_t &w, const int &m, const int &c, + const scalar_t &top_grad, + const scalar_t &attn_weight, + scalar_t* &grad_value, + scalar_t* grad_sampling_loc, + scalar_t* grad_attn_weight) +{ + const int h_low = floor(h); + const int w_low = floor(w); + const int h_high = h_low + 1; + const int w_high = w_low + 1; + + const scalar_t lh = h - h_low; + const scalar_t lw = w - w_low; + const scalar_t hh = 1 - lh, hw = 1 - lw; + + const int w_stride = nheads * channels; + const int h_stride = width * w_stride; + const int h_low_ptr_offset = h_low * h_stride; + const int h_high_ptr_offset = h_low_ptr_offset + h_stride; + const int w_low_ptr_offset = w_low * w_stride; + const int w_high_ptr_offset = w_low_ptr_offset + w_stride; + const int base_ptr = m * channels + c; + + const scalar_t w1 = hh * hw, w2 = hh * lw, w3 = lh * hw, w4 = lh * lw; + const scalar_t top_grad_value = top_grad * attn_weight; + scalar_t grad_h_weight = 0, grad_w_weight = 0; + + scalar_t v1 = 0; + if (h_low >= 0 && w_low >= 0) + { + const int ptr1 = h_low_ptr_offset + w_low_ptr_offset + base_ptr; + v1 = bottom_data[ptr1]; + grad_h_weight -= hw * v1; + grad_w_weight -= hh * v1; + atomicAdd(grad_value+ptr1, w1*top_grad_value); + } + scalar_t v2 = 0; + if (h_low >= 0 && w_high <= width - 1) + { + const int ptr2 = h_low_ptr_offset + w_high_ptr_offset + base_ptr; + v2 = bottom_data[ptr2]; + grad_h_weight -= lw * v2; + grad_w_weight += hh * v2; + atomicAdd(grad_value+ptr2, w2*top_grad_value); + } + scalar_t v3 = 0; + if (h_high <= height - 1 && w_low >= 0) + { + const int ptr3 = h_high_ptr_offset + w_low_ptr_offset + base_ptr; + v3 = bottom_data[ptr3]; + grad_h_weight += hw * v3; + grad_w_weight -= lh * v3; + atomicAdd(grad_value+ptr3, w3*top_grad_value); + } + scalar_t v4 = 0; + if (h_high <= height - 1 && w_high <= width - 1) + { + const int ptr4 = h_high_ptr_offset + w_high_ptr_offset + base_ptr; + v4 = bottom_data[ptr4]; + grad_h_weight += lw * v4; + grad_w_weight += lh * v4; + atomicAdd(grad_value+ptr4, w4*top_grad_value); + } + + const scalar_t val = (w1 * v1 + w2 * v2 + w3 * v3 + w4 * v4); + atomicAdd(grad_attn_weight, top_grad * val); + atomicAdd(grad_sampling_loc, width * grad_w_weight * top_grad_value); + atomicAdd(grad_sampling_loc + 1, height * grad_h_weight * top_grad_value); +} + + +template +__global__ void ms_deformable_im2col_gpu_kernel(const int n, + const scalar_t *data_value, + const int64_t *data_spatial_shapes, + const int64_t *data_level_start_index, + const scalar_t *data_sampling_loc, + const scalar_t *data_attn_weight, + const int batch_size, + const int spatial_size, + const int num_heads, + const int channels, + const int num_levels, + const int num_query, + const int num_point, + scalar_t *data_col) +{ + CUDA_KERNEL_LOOP(index, n) + { + int _temp = index; + const int c_col = _temp % channels; + _temp /= channels; + const int sampling_index = _temp; + const int m_col = _temp % num_heads; + _temp /= num_heads; + const int q_col = _temp % num_query; + _temp /= num_query; + const int b_col = _temp; + + scalar_t *data_col_ptr = data_col + index; + int data_weight_ptr = sampling_index * num_levels * num_point; + int data_loc_w_ptr = data_weight_ptr << 1; + const int qid_stride = num_heads * channels; + const int data_value_ptr_init_offset = b_col * spatial_size * qid_stride; + scalar_t col = 0; + + for (int l_col=0; l_col < num_levels; ++l_col) + { + const int level_start_id = data_level_start_index[l_col]; + const int spatial_h_ptr = l_col << 1; + const int spatial_h = data_spatial_shapes[spatial_h_ptr]; + const int spatial_w = data_spatial_shapes[spatial_h_ptr + 1]; + const scalar_t *data_value_ptr = data_value + (data_value_ptr_init_offset + level_start_id * qid_stride); + for (int p_col=0; p_col < num_point; ++p_col) + { + const scalar_t loc_w = data_sampling_loc[data_loc_w_ptr]; + const scalar_t loc_h = data_sampling_loc[data_loc_w_ptr + 1]; + const scalar_t weight = data_attn_weight[data_weight_ptr]; + + const scalar_t h_im = loc_h * spatial_h - 0.5; + const scalar_t w_im = loc_w * spatial_w - 0.5; + + if (h_im > -1 && w_im > -1 && h_im < spatial_h && w_im < spatial_w) + { + col += ms_deform_attn_im2col_bilinear(data_value_ptr, spatial_h, spatial_w, num_heads, channels, h_im, w_im, m_col, c_col) * weight; + } + + data_weight_ptr += 1; + data_loc_w_ptr += 2; + } + } + *data_col_ptr = col; + } +} + +template +__global__ void ms_deformable_col2im_gpu_kernel_shm_blocksize_aware_reduce_v1(const int n, + const scalar_t *grad_col, + const scalar_t *data_value, + const int64_t *data_spatial_shapes, + const int64_t *data_level_start_index, + const scalar_t *data_sampling_loc, + const scalar_t *data_attn_weight, + const int batch_size, + const int spatial_size, + const int num_heads, + const int channels, + const int num_levels, + const int num_query, + const int num_point, + scalar_t *grad_value, + scalar_t *grad_sampling_loc, + scalar_t *grad_attn_weight) +{ + CUDA_KERNEL_LOOP(index, n) + { + __shared__ scalar_t cache_grad_sampling_loc[blockSize * 2]; + __shared__ scalar_t cache_grad_attn_weight[blockSize]; + unsigned int tid = threadIdx.x; + int _temp = index; + const int c_col = _temp % channels; + _temp /= channels; + const int sampling_index = _temp; + const int m_col = _temp % num_heads; + _temp /= num_heads; + const int q_col = _temp % num_query; + _temp /= num_query; + const int b_col = _temp; + + const scalar_t top_grad = grad_col[index]; + + int data_weight_ptr = sampling_index * num_levels * num_point; + int data_loc_w_ptr = data_weight_ptr << 1; + const int grad_sampling_ptr = data_weight_ptr; + grad_sampling_loc += grad_sampling_ptr << 1; + grad_attn_weight += grad_sampling_ptr; + const int grad_weight_stride = 1; + const int grad_loc_stride = 2; + const int qid_stride = num_heads * channels; + const int data_value_ptr_init_offset = b_col * spatial_size * qid_stride; + + for (int l_col=0; l_col < num_levels; ++l_col) + { + const int level_start_id = data_level_start_index[l_col]; + const int spatial_h_ptr = l_col << 1; + const int spatial_h = data_spatial_shapes[spatial_h_ptr]; + const int spatial_w = data_spatial_shapes[spatial_h_ptr + 1]; + const int value_ptr_offset = data_value_ptr_init_offset + level_start_id * qid_stride; + const scalar_t *data_value_ptr = data_value + value_ptr_offset; + scalar_t *grad_value_ptr = grad_value + value_ptr_offset; + + for (int p_col=0; p_col < num_point; ++p_col) + { + const scalar_t loc_w = data_sampling_loc[data_loc_w_ptr]; + const scalar_t loc_h = data_sampling_loc[data_loc_w_ptr + 1]; + const scalar_t weight = data_attn_weight[data_weight_ptr]; + + const scalar_t h_im = loc_h * spatial_h - 0.5; + const scalar_t w_im = loc_w * spatial_w - 0.5; + *(cache_grad_sampling_loc+(threadIdx.x << 1)) = 0; + *(cache_grad_sampling_loc+((threadIdx.x << 1) + 1)) = 0; + *(cache_grad_attn_weight+threadIdx.x)=0; + if (h_im > -1 && w_im > -1 && h_im < spatial_h && w_im < spatial_w) + { + ms_deform_attn_col2im_bilinear( + data_value_ptr, spatial_h, spatial_w, num_heads, channels, h_im, w_im, m_col, c_col, + top_grad, weight, grad_value_ptr, + cache_grad_sampling_loc+(threadIdx.x << 1), cache_grad_attn_weight+threadIdx.x); + } + + __syncthreads(); + if (tid == 0) + { + scalar_t _grad_w=cache_grad_sampling_loc[0], _grad_h=cache_grad_sampling_loc[1], _grad_a=cache_grad_attn_weight[0]; + int sid=2; + for (unsigned int tid = 1; tid < blockSize; ++tid) + { + _grad_w += cache_grad_sampling_loc[sid]; + _grad_h += cache_grad_sampling_loc[sid + 1]; + _grad_a += cache_grad_attn_weight[tid]; + sid += 2; + } + + + *grad_sampling_loc = _grad_w; + *(grad_sampling_loc + 1) = _grad_h; + *grad_attn_weight = _grad_a; + } + __syncthreads(); + + data_weight_ptr += 1; + data_loc_w_ptr += 2; + grad_attn_weight += grad_weight_stride; + grad_sampling_loc += grad_loc_stride; + } + } + } +} + + +template +__global__ void ms_deformable_col2im_gpu_kernel_shm_blocksize_aware_reduce_v2(const int n, + const scalar_t *grad_col, + const scalar_t *data_value, + const int64_t *data_spatial_shapes, + const int64_t *data_level_start_index, + const scalar_t *data_sampling_loc, + const scalar_t *data_attn_weight, + const int batch_size, + const int spatial_size, + const int num_heads, + const int channels, + const int num_levels, + const int num_query, + const int num_point, + scalar_t *grad_value, + scalar_t *grad_sampling_loc, + scalar_t *grad_attn_weight) +{ + CUDA_KERNEL_LOOP(index, n) + { + __shared__ scalar_t cache_grad_sampling_loc[blockSize * 2]; + __shared__ scalar_t cache_grad_attn_weight[blockSize]; + unsigned int tid = threadIdx.x; + int _temp = index; + const int c_col = _temp % channels; + _temp /= channels; + const int sampling_index = _temp; + const int m_col = _temp % num_heads; + _temp /= num_heads; + const int q_col = _temp % num_query; + _temp /= num_query; + const int b_col = _temp; + + const scalar_t top_grad = grad_col[index]; + + int data_weight_ptr = sampling_index * num_levels * num_point; + int data_loc_w_ptr = data_weight_ptr << 1; + const int grad_sampling_ptr = data_weight_ptr; + grad_sampling_loc += grad_sampling_ptr << 1; + grad_attn_weight += grad_sampling_ptr; + const int grad_weight_stride = 1; + const int grad_loc_stride = 2; + const int qid_stride = num_heads * channels; + const int data_value_ptr_init_offset = b_col * spatial_size * qid_stride; + + for (int l_col=0; l_col < num_levels; ++l_col) + { + const int level_start_id = data_level_start_index[l_col]; + const int spatial_h_ptr = l_col << 1; + const int spatial_h = data_spatial_shapes[spatial_h_ptr]; + const int spatial_w = data_spatial_shapes[spatial_h_ptr + 1]; + const int value_ptr_offset = data_value_ptr_init_offset + level_start_id * qid_stride; + const scalar_t *data_value_ptr = data_value + value_ptr_offset; + scalar_t *grad_value_ptr = grad_value + value_ptr_offset; + + for (int p_col=0; p_col < num_point; ++p_col) + { + const scalar_t loc_w = data_sampling_loc[data_loc_w_ptr]; + const scalar_t loc_h = data_sampling_loc[data_loc_w_ptr + 1]; + const scalar_t weight = data_attn_weight[data_weight_ptr]; + + const scalar_t h_im = loc_h * spatial_h - 0.5; + const scalar_t w_im = loc_w * spatial_w - 0.5; + *(cache_grad_sampling_loc+(threadIdx.x << 1)) = 0; + *(cache_grad_sampling_loc+((threadIdx.x << 1) + 1)) = 0; + *(cache_grad_attn_weight+threadIdx.x)=0; + if (h_im > -1 && w_im > -1 && h_im < spatial_h && w_im < spatial_w) + { + ms_deform_attn_col2im_bilinear( + data_value_ptr, spatial_h, spatial_w, num_heads, channels, h_im, w_im, m_col, c_col, + top_grad, weight, grad_value_ptr, + cache_grad_sampling_loc+(threadIdx.x << 1), cache_grad_attn_weight+threadIdx.x); + } + + __syncthreads(); + + for (unsigned int s=blockSize/2; s>0; s>>=1) + { + if (tid < s) { + const unsigned int xid1 = tid << 1; + const unsigned int xid2 = (tid + s) << 1; + cache_grad_attn_weight[tid] += cache_grad_attn_weight[tid + s]; + cache_grad_sampling_loc[xid1] += cache_grad_sampling_loc[xid2]; + cache_grad_sampling_loc[xid1 + 1] += cache_grad_sampling_loc[xid2 + 1]; + } + __syncthreads(); + } + + if (tid == 0) + { + *grad_sampling_loc = cache_grad_sampling_loc[0]; + *(grad_sampling_loc + 1) = cache_grad_sampling_loc[1]; + *grad_attn_weight = cache_grad_attn_weight[0]; + } + __syncthreads(); + + data_weight_ptr += 1; + data_loc_w_ptr += 2; + grad_attn_weight += grad_weight_stride; + grad_sampling_loc += grad_loc_stride; + } + } + } +} + + +template +__global__ void ms_deformable_col2im_gpu_kernel_shm_reduce_v1(const int n, + const scalar_t *grad_col, + const scalar_t *data_value, + const int64_t *data_spatial_shapes, + const int64_t *data_level_start_index, + const scalar_t *data_sampling_loc, + const scalar_t *data_attn_weight, + const int batch_size, + const int spatial_size, + const int num_heads, + const int channels, + const int num_levels, + const int num_query, + const int num_point, + scalar_t *grad_value, + scalar_t *grad_sampling_loc, + scalar_t *grad_attn_weight) +{ + CUDA_KERNEL_LOOP(index, n) + { + extern __shared__ int _s[]; + scalar_t* cache_grad_sampling_loc = (scalar_t*)_s; + scalar_t* cache_grad_attn_weight = cache_grad_sampling_loc + 2 * blockDim.x; + unsigned int tid = threadIdx.x; + int _temp = index; + const int c_col = _temp % channels; + _temp /= channels; + const int sampling_index = _temp; + const int m_col = _temp % num_heads; + _temp /= num_heads; + const int q_col = _temp % num_query; + _temp /= num_query; + const int b_col = _temp; + + const scalar_t top_grad = grad_col[index]; + + int data_weight_ptr = sampling_index * num_levels * num_point; + int data_loc_w_ptr = data_weight_ptr << 1; + const int grad_sampling_ptr = data_weight_ptr; + grad_sampling_loc += grad_sampling_ptr << 1; + grad_attn_weight += grad_sampling_ptr; + const int grad_weight_stride = 1; + const int grad_loc_stride = 2; + const int qid_stride = num_heads * channels; + const int data_value_ptr_init_offset = b_col * spatial_size * qid_stride; + + for (int l_col=0; l_col < num_levels; ++l_col) + { + const int level_start_id = data_level_start_index[l_col]; + const int spatial_h_ptr = l_col << 1; + const int spatial_h = data_spatial_shapes[spatial_h_ptr]; + const int spatial_w = data_spatial_shapes[spatial_h_ptr + 1]; + const int value_ptr_offset = data_value_ptr_init_offset + level_start_id * qid_stride; + const scalar_t *data_value_ptr = data_value + value_ptr_offset; + scalar_t *grad_value_ptr = grad_value + value_ptr_offset; + + for (int p_col=0; p_col < num_point; ++p_col) + { + const scalar_t loc_w = data_sampling_loc[data_loc_w_ptr]; + const scalar_t loc_h = data_sampling_loc[data_loc_w_ptr + 1]; + const scalar_t weight = data_attn_weight[data_weight_ptr]; + + const scalar_t h_im = loc_h * spatial_h - 0.5; + const scalar_t w_im = loc_w * spatial_w - 0.5; + *(cache_grad_sampling_loc+(threadIdx.x << 1)) = 0; + *(cache_grad_sampling_loc+((threadIdx.x << 1) + 1)) = 0; + *(cache_grad_attn_weight+threadIdx.x)=0; + if (h_im > -1 && w_im > -1 && h_im < spatial_h && w_im < spatial_w) + { + ms_deform_attn_col2im_bilinear( + data_value_ptr, spatial_h, spatial_w, num_heads, channels, h_im, w_im, m_col, c_col, + top_grad, weight, grad_value_ptr, + cache_grad_sampling_loc+(threadIdx.x << 1), cache_grad_attn_weight+threadIdx.x); + } + + __syncthreads(); + if (tid == 0) + { + scalar_t _grad_w=cache_grad_sampling_loc[0], _grad_h=cache_grad_sampling_loc[1], _grad_a=cache_grad_attn_weight[0]; + int sid=2; + for (unsigned int tid = 1; tid < blockDim.x; ++tid) + { + _grad_w += cache_grad_sampling_loc[sid]; + _grad_h += cache_grad_sampling_loc[sid + 1]; + _grad_a += cache_grad_attn_weight[tid]; + sid += 2; + } + + + *grad_sampling_loc = _grad_w; + *(grad_sampling_loc + 1) = _grad_h; + *grad_attn_weight = _grad_a; + } + __syncthreads(); + + data_weight_ptr += 1; + data_loc_w_ptr += 2; + grad_attn_weight += grad_weight_stride; + grad_sampling_loc += grad_loc_stride; + } + } + } +} + +template +__global__ void ms_deformable_col2im_gpu_kernel_shm_reduce_v2(const int n, + const scalar_t *grad_col, + const scalar_t *data_value, + const int64_t *data_spatial_shapes, + const int64_t *data_level_start_index, + const scalar_t *data_sampling_loc, + const scalar_t *data_attn_weight, + const int batch_size, + const int spatial_size, + const int num_heads, + const int channels, + const int num_levels, + const int num_query, + const int num_point, + scalar_t *grad_value, + scalar_t *grad_sampling_loc, + scalar_t *grad_attn_weight) +{ + CUDA_KERNEL_LOOP(index, n) + { + extern __shared__ int _s[]; + scalar_t* cache_grad_sampling_loc = (scalar_t*)_s; + scalar_t* cache_grad_attn_weight = cache_grad_sampling_loc + 2 * blockDim.x; + unsigned int tid = threadIdx.x; + int _temp = index; + const int c_col = _temp % channels; + _temp /= channels; + const int sampling_index = _temp; + const int m_col = _temp % num_heads; + _temp /= num_heads; + const int q_col = _temp % num_query; + _temp /= num_query; + const int b_col = _temp; + + const scalar_t top_grad = grad_col[index]; + + int data_weight_ptr = sampling_index * num_levels * num_point; + int data_loc_w_ptr = data_weight_ptr << 1; + const int grad_sampling_ptr = data_weight_ptr; + grad_sampling_loc += grad_sampling_ptr << 1; + grad_attn_weight += grad_sampling_ptr; + const int grad_weight_stride = 1; + const int grad_loc_stride = 2; + const int qid_stride = num_heads * channels; + const int data_value_ptr_init_offset = b_col * spatial_size * qid_stride; + + for (int l_col=0; l_col < num_levels; ++l_col) + { + const int level_start_id = data_level_start_index[l_col]; + const int spatial_h_ptr = l_col << 1; + const int spatial_h = data_spatial_shapes[spatial_h_ptr]; + const int spatial_w = data_spatial_shapes[spatial_h_ptr + 1]; + const int value_ptr_offset = data_value_ptr_init_offset + level_start_id * qid_stride; + const scalar_t *data_value_ptr = data_value + value_ptr_offset; + scalar_t *grad_value_ptr = grad_value + value_ptr_offset; + + for (int p_col=0; p_col < num_point; ++p_col) + { + const scalar_t loc_w = data_sampling_loc[data_loc_w_ptr]; + const scalar_t loc_h = data_sampling_loc[data_loc_w_ptr + 1]; + const scalar_t weight = data_attn_weight[data_weight_ptr]; + + const scalar_t h_im = loc_h * spatial_h - 0.5; + const scalar_t w_im = loc_w * spatial_w - 0.5; + *(cache_grad_sampling_loc+(threadIdx.x << 1)) = 0; + *(cache_grad_sampling_loc+((threadIdx.x << 1) + 1)) = 0; + *(cache_grad_attn_weight+threadIdx.x)=0; + if (h_im > -1 && w_im > -1 && h_im < spatial_h && w_im < spatial_w) + { + ms_deform_attn_col2im_bilinear( + data_value_ptr, spatial_h, spatial_w, num_heads, channels, h_im, w_im, m_col, c_col, + top_grad, weight, grad_value_ptr, + cache_grad_sampling_loc+(threadIdx.x << 1), cache_grad_attn_weight+threadIdx.x); + } + + __syncthreads(); + + for (unsigned int s=blockDim.x/2, spre=blockDim.x; s>0; s>>=1, spre>>=1) + { + if (tid < s) { + const unsigned int xid1 = tid << 1; + const unsigned int xid2 = (tid + s) << 1; + cache_grad_attn_weight[tid] += cache_grad_attn_weight[tid + s]; + cache_grad_sampling_loc[xid1] += cache_grad_sampling_loc[xid2]; + cache_grad_sampling_loc[xid1 + 1] += cache_grad_sampling_loc[xid2 + 1]; + if (tid + (s << 1) < spre) + { + cache_grad_attn_weight[tid] += cache_grad_attn_weight[tid + (s << 1)]; + cache_grad_sampling_loc[xid1] += cache_grad_sampling_loc[xid2 + (s << 1)]; + cache_grad_sampling_loc[xid1 + 1] += cache_grad_sampling_loc[xid2 + 1 + (s << 1)]; + } + } + __syncthreads(); + } + + if (tid == 0) + { + *grad_sampling_loc = cache_grad_sampling_loc[0]; + *(grad_sampling_loc + 1) = cache_grad_sampling_loc[1]; + *grad_attn_weight = cache_grad_attn_weight[0]; + } + __syncthreads(); + + data_weight_ptr += 1; + data_loc_w_ptr += 2; + grad_attn_weight += grad_weight_stride; + grad_sampling_loc += grad_loc_stride; + } + } + } +} + +template +__global__ void ms_deformable_col2im_gpu_kernel_shm_reduce_v2_multi_blocks(const int n, + const scalar_t *grad_col, + const scalar_t *data_value, + const int64_t *data_spatial_shapes, + const int64_t *data_level_start_index, + const scalar_t *data_sampling_loc, + const scalar_t *data_attn_weight, + const int batch_size, + const int spatial_size, + const int num_heads, + const int channels, + const int num_levels, + const int num_query, + const int num_point, + scalar_t *grad_value, + scalar_t *grad_sampling_loc, + scalar_t *grad_attn_weight) +{ + CUDA_KERNEL_LOOP(index, n) + { + extern __shared__ int _s[]; + scalar_t* cache_grad_sampling_loc = (scalar_t*)_s; + scalar_t* cache_grad_attn_weight = cache_grad_sampling_loc + 2 * blockDim.x; + unsigned int tid = threadIdx.x; + int _temp = index; + const int c_col = _temp % channels; + _temp /= channels; + const int sampling_index = _temp; + const int m_col = _temp % num_heads; + _temp /= num_heads; + const int q_col = _temp % num_query; + _temp /= num_query; + const int b_col = _temp; + + const scalar_t top_grad = grad_col[index]; + + int data_weight_ptr = sampling_index * num_levels * num_point; + int data_loc_w_ptr = data_weight_ptr << 1; + const int grad_sampling_ptr = data_weight_ptr; + grad_sampling_loc += grad_sampling_ptr << 1; + grad_attn_weight += grad_sampling_ptr; + const int grad_weight_stride = 1; + const int grad_loc_stride = 2; + const int qid_stride = num_heads * channels; + const int data_value_ptr_init_offset = b_col * spatial_size * qid_stride; + + for (int l_col=0; l_col < num_levels; ++l_col) + { + const int level_start_id = data_level_start_index[l_col]; + const int spatial_h_ptr = l_col << 1; + const int spatial_h = data_spatial_shapes[spatial_h_ptr]; + const int spatial_w = data_spatial_shapes[spatial_h_ptr + 1]; + const int value_ptr_offset = data_value_ptr_init_offset + level_start_id * qid_stride; + const scalar_t *data_value_ptr = data_value + value_ptr_offset; + scalar_t *grad_value_ptr = grad_value + value_ptr_offset; + + for (int p_col=0; p_col < num_point; ++p_col) + { + const scalar_t loc_w = data_sampling_loc[data_loc_w_ptr]; + const scalar_t loc_h = data_sampling_loc[data_loc_w_ptr + 1]; + const scalar_t weight = data_attn_weight[data_weight_ptr]; + + const scalar_t h_im = loc_h * spatial_h - 0.5; + const scalar_t w_im = loc_w * spatial_w - 0.5; + *(cache_grad_sampling_loc+(threadIdx.x << 1)) = 0; + *(cache_grad_sampling_loc+((threadIdx.x << 1) + 1)) = 0; + *(cache_grad_attn_weight+threadIdx.x)=0; + if (h_im > -1 && w_im > -1 && h_im < spatial_h && w_im < spatial_w) + { + ms_deform_attn_col2im_bilinear( + data_value_ptr, spatial_h, spatial_w, num_heads, channels, h_im, w_im, m_col, c_col, + top_grad, weight, grad_value_ptr, + cache_grad_sampling_loc+(threadIdx.x << 1), cache_grad_attn_weight+threadIdx.x); + } + + __syncthreads(); + + for (unsigned int s=blockDim.x/2, spre=blockDim.x; s>0; s>>=1, spre>>=1) + { + if (tid < s) { + const unsigned int xid1 = tid << 1; + const unsigned int xid2 = (tid + s) << 1; + cache_grad_attn_weight[tid] += cache_grad_attn_weight[tid + s]; + cache_grad_sampling_loc[xid1] += cache_grad_sampling_loc[xid2]; + cache_grad_sampling_loc[xid1 + 1] += cache_grad_sampling_loc[xid2 + 1]; + if (tid + (s << 1) < spre) + { + cache_grad_attn_weight[tid] += cache_grad_attn_weight[tid + (s << 1)]; + cache_grad_sampling_loc[xid1] += cache_grad_sampling_loc[xid2 + (s << 1)]; + cache_grad_sampling_loc[xid1 + 1] += cache_grad_sampling_loc[xid2 + 1 + (s << 1)]; + } + } + __syncthreads(); + } + + if (tid == 0) + { + atomicAdd(grad_sampling_loc, cache_grad_sampling_loc[0]); + atomicAdd(grad_sampling_loc + 1, cache_grad_sampling_loc[1]); + atomicAdd(grad_attn_weight, cache_grad_attn_weight[0]); + } + __syncthreads(); + + data_weight_ptr += 1; + data_loc_w_ptr += 2; + grad_attn_weight += grad_weight_stride; + grad_sampling_loc += grad_loc_stride; + } + } + } +} + + +template +__global__ void ms_deformable_col2im_gpu_kernel_gm(const int n, + const scalar_t *grad_col, + const scalar_t *data_value, + const int64_t *data_spatial_shapes, + const int64_t *data_level_start_index, + const scalar_t *data_sampling_loc, + const scalar_t *data_attn_weight, + const int batch_size, + const int spatial_size, + const int num_heads, + const int channels, + const int num_levels, + const int num_query, + const int num_point, + scalar_t *grad_value, + scalar_t *grad_sampling_loc, + scalar_t *grad_attn_weight) +{ + CUDA_KERNEL_LOOP(index, n) + { + int _temp = index; + const int c_col = _temp % channels; + _temp /= channels; + const int sampling_index = _temp; + const int m_col = _temp % num_heads; + _temp /= num_heads; + const int q_col = _temp % num_query; + _temp /= num_query; + const int b_col = _temp; + + const scalar_t top_grad = grad_col[index]; + + int data_weight_ptr = sampling_index * num_levels * num_point; + int data_loc_w_ptr = data_weight_ptr << 1; + const int grad_sampling_ptr = data_weight_ptr; + grad_sampling_loc += grad_sampling_ptr << 1; + grad_attn_weight += grad_sampling_ptr; + const int grad_weight_stride = 1; + const int grad_loc_stride = 2; + const int qid_stride = num_heads * channels; + const int data_value_ptr_init_offset = b_col * spatial_size * qid_stride; + + for (int l_col=0; l_col < num_levels; ++l_col) + { + const int level_start_id = data_level_start_index[l_col]; + const int spatial_h_ptr = l_col << 1; + const int spatial_h = data_spatial_shapes[spatial_h_ptr]; + const int spatial_w = data_spatial_shapes[spatial_h_ptr + 1]; + const int value_ptr_offset = data_value_ptr_init_offset + level_start_id * qid_stride; + const scalar_t *data_value_ptr = data_value + value_ptr_offset; + scalar_t *grad_value_ptr = grad_value + value_ptr_offset; + + for (int p_col=0; p_col < num_point; ++p_col) + { + const scalar_t loc_w = data_sampling_loc[data_loc_w_ptr]; + const scalar_t loc_h = data_sampling_loc[data_loc_w_ptr + 1]; + const scalar_t weight = data_attn_weight[data_weight_ptr]; + + const scalar_t h_im = loc_h * spatial_h - 0.5; + const scalar_t w_im = loc_w * spatial_w - 0.5; + if (h_im > -1 && w_im > -1 && h_im < spatial_h && w_im < spatial_w) + { + ms_deform_attn_col2im_bilinear_gm( + data_value_ptr, spatial_h, spatial_w, num_heads, channels, h_im, w_im, m_col, c_col, + top_grad, weight, grad_value_ptr, + grad_sampling_loc, grad_attn_weight); + } + data_weight_ptr += 1; + data_loc_w_ptr += 2; + grad_attn_weight += grad_weight_stride; + grad_sampling_loc += grad_loc_stride; + } + } + } +} + + +template +void ms_deformable_im2col_cuda(cudaStream_t stream, + const scalar_t* data_value, + const int64_t* data_spatial_shapes, + const int64_t* data_level_start_index, + const scalar_t* data_sampling_loc, + const scalar_t* data_attn_weight, + const int batch_size, + const int spatial_size, + const int num_heads, + const int channels, + const int num_levels, + const int num_query, + const int num_point, + scalar_t* data_col) +{ + const int num_kernels = batch_size * num_query * num_heads * channels; + const int num_actual_kernels = batch_size * num_query * num_heads * channels; + const int num_threads = CUDA_NUM_THREADS; + ms_deformable_im2col_gpu_kernel + <<>>( + num_kernels, data_value, data_spatial_shapes, data_level_start_index, data_sampling_loc, data_attn_weight, + batch_size, spatial_size, num_heads, channels, num_levels, num_query, num_point, data_col); + + cudaError_t err = cudaGetLastError(); + if (err != cudaSuccess) + { + printf("error in ms_deformable_im2col_cuda: %s\n", cudaGetErrorString(err)); + } + +} + +template +void ms_deformable_col2im_cuda(cudaStream_t stream, + const scalar_t* grad_col, + const scalar_t* data_value, + const int64_t * data_spatial_shapes, + const int64_t * data_level_start_index, + const scalar_t * data_sampling_loc, + const scalar_t * data_attn_weight, + const int batch_size, + const int spatial_size, + const int num_heads, + const int channels, + const int num_levels, + const int num_query, + const int num_point, + scalar_t* grad_value, + scalar_t* grad_sampling_loc, + scalar_t* grad_attn_weight) +{ + const int num_threads = (channels > CUDA_NUM_THREADS)?CUDA_NUM_THREADS:channels; + const int num_kernels = batch_size * num_query * num_heads * channels; + const int num_actual_kernels = batch_size * num_query * num_heads * channels; + if (channels > 1024) + { + if ((channels & 1023) == 0) + { + ms_deformable_col2im_gpu_kernel_shm_reduce_v2_multi_blocks + <<>>( + num_kernels, + grad_col, + data_value, + data_spatial_shapes, + data_level_start_index, + data_sampling_loc, + data_attn_weight, + batch_size, + spatial_size, + num_heads, + channels, + num_levels, + num_query, + num_point, + grad_value, + grad_sampling_loc, + grad_attn_weight); + } + else + { + ms_deformable_col2im_gpu_kernel_gm + <<>>( + num_kernels, + grad_col, + data_value, + data_spatial_shapes, + data_level_start_index, + data_sampling_loc, + data_attn_weight, + batch_size, + spatial_size, + num_heads, + channels, + num_levels, + num_query, + num_point, + grad_value, + grad_sampling_loc, + grad_attn_weight); + } + } + else{ + switch(channels) + { + case 1: + ms_deformable_col2im_gpu_kernel_shm_blocksize_aware_reduce_v1 + <<>>( + num_kernels, + grad_col, + data_value, + data_spatial_shapes, + data_level_start_index, + data_sampling_loc, + data_attn_weight, + batch_size, + spatial_size, + num_heads, + channels, + num_levels, + num_query, + num_point, + grad_value, + grad_sampling_loc, + grad_attn_weight); + break; + case 2: + ms_deformable_col2im_gpu_kernel_shm_blocksize_aware_reduce_v1 + <<>>( + num_kernels, + grad_col, + data_value, + data_spatial_shapes, + data_level_start_index, + data_sampling_loc, + data_attn_weight, + batch_size, + spatial_size, + num_heads, + channels, + num_levels, + num_query, + num_point, + grad_value, + grad_sampling_loc, + grad_attn_weight); + break; + case 4: + ms_deformable_col2im_gpu_kernel_shm_blocksize_aware_reduce_v1 + <<>>( + num_kernels, + grad_col, + data_value, + data_spatial_shapes, + data_level_start_index, + data_sampling_loc, + data_attn_weight, + batch_size, + spatial_size, + num_heads, + channels, + num_levels, + num_query, + num_point, + grad_value, + grad_sampling_loc, + grad_attn_weight); + break; + case 8: + ms_deformable_col2im_gpu_kernel_shm_blocksize_aware_reduce_v1 + <<>>( + num_kernels, + grad_col, + data_value, + data_spatial_shapes, + data_level_start_index, + data_sampling_loc, + data_attn_weight, + batch_size, + spatial_size, + num_heads, + channels, + num_levels, + num_query, + num_point, + grad_value, + grad_sampling_loc, + grad_attn_weight); + break; + case 16: + ms_deformable_col2im_gpu_kernel_shm_blocksize_aware_reduce_v1 + <<>>( + num_kernels, + grad_col, + data_value, + data_spatial_shapes, + data_level_start_index, + data_sampling_loc, + data_attn_weight, + batch_size, + spatial_size, + num_heads, + channels, + num_levels, + num_query, + num_point, + grad_value, + grad_sampling_loc, + grad_attn_weight); + break; + case 32: + ms_deformable_col2im_gpu_kernel_shm_blocksize_aware_reduce_v1 + <<>>( + num_kernels, + grad_col, + data_value, + data_spatial_shapes, + data_level_start_index, + data_sampling_loc, + data_attn_weight, + batch_size, + spatial_size, + num_heads, + channels, + num_levels, + num_query, + num_point, + grad_value, + grad_sampling_loc, + grad_attn_weight); + break; + case 64: + ms_deformable_col2im_gpu_kernel_shm_blocksize_aware_reduce_v2 + <<>>( + num_kernels, + grad_col, + data_value, + data_spatial_shapes, + data_level_start_index, + data_sampling_loc, + data_attn_weight, + batch_size, + spatial_size, + num_heads, + channels, + num_levels, + num_query, + num_point, + grad_value, + grad_sampling_loc, + grad_attn_weight); + break; + case 128: + ms_deformable_col2im_gpu_kernel_shm_blocksize_aware_reduce_v2 + <<>>( + num_kernels, + grad_col, + data_value, + data_spatial_shapes, + data_level_start_index, + data_sampling_loc, + data_attn_weight, + batch_size, + spatial_size, + num_heads, + channels, + num_levels, + num_query, + num_point, + grad_value, + grad_sampling_loc, + grad_attn_weight); + break; + case 256: + ms_deformable_col2im_gpu_kernel_shm_blocksize_aware_reduce_v2 + <<>>( + num_kernels, + grad_col, + data_value, + data_spatial_shapes, + data_level_start_index, + data_sampling_loc, + data_attn_weight, + batch_size, + spatial_size, + num_heads, + channels, + num_levels, + num_query, + num_point, + grad_value, + grad_sampling_loc, + grad_attn_weight); + break; + case 512: + ms_deformable_col2im_gpu_kernel_shm_blocksize_aware_reduce_v2 + <<>>( + num_kernels, + grad_col, + data_value, + data_spatial_shapes, + data_level_start_index, + data_sampling_loc, + data_attn_weight, + batch_size, + spatial_size, + num_heads, + channels, + num_levels, + num_query, + num_point, + grad_value, + grad_sampling_loc, + grad_attn_weight); + break; + case 1024: + ms_deformable_col2im_gpu_kernel_shm_blocksize_aware_reduce_v2 + <<>>( + num_kernels, + grad_col, + data_value, + data_spatial_shapes, + data_level_start_index, + data_sampling_loc, + data_attn_weight, + batch_size, + spatial_size, + num_heads, + channels, + num_levels, + num_query, + num_point, + grad_value, + grad_sampling_loc, + grad_attn_weight); + break; + default: + if (channels < 64) + { + ms_deformable_col2im_gpu_kernel_shm_reduce_v1 + <<>>( + num_kernels, + grad_col, + data_value, + data_spatial_shapes, + data_level_start_index, + data_sampling_loc, + data_attn_weight, + batch_size, + spatial_size, + num_heads, + channels, + num_levels, + num_query, + num_point, + grad_value, + grad_sampling_loc, + grad_attn_weight); + } + else + { + ms_deformable_col2im_gpu_kernel_shm_reduce_v2 + <<>>( + num_kernels, + grad_col, + data_value, + data_spatial_shapes, + data_level_start_index, + data_sampling_loc, + data_attn_weight, + batch_size, + spatial_size, + num_heads, + channels, + num_levels, + num_query, + num_point, + grad_value, + grad_sampling_loc, + grad_attn_weight); + } + } + } + cudaError_t err = cudaGetLastError(); + if (err != cudaSuccess) + { + printf("error in ms_deformable_col2im_cuda: %s\n", cudaGetErrorString(err)); + } + +} \ No newline at end of file diff --git a/models/GroundingDINO/ops/src/ms_deform_attn.h b/models/GroundingDINO/ops/src/ms_deform_attn.h new file mode 100644 index 0000000000000000000000000000000000000000..ac0ef2ec25f7d0ee51ca2d807b159ddf85652017 --- /dev/null +++ b/models/GroundingDINO/ops/src/ms_deform_attn.h @@ -0,0 +1,62 @@ +/*! +************************************************************************************************** +* Deformable DETR +* Copyright (c) 2020 SenseTime. All Rights Reserved. +* Licensed under the Apache License, Version 2.0 [see LICENSE for details] +************************************************************************************************** +* Modified from https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 +************************************************************************************************** +*/ + +#pragma once + +#include "cpu/ms_deform_attn_cpu.h" + +#ifdef WITH_CUDA +#include "cuda/ms_deform_attn_cuda.h" +#endif + + +at::Tensor +ms_deform_attn_forward( + const at::Tensor &value, + const at::Tensor &spatial_shapes, + const at::Tensor &level_start_index, + const at::Tensor &sampling_loc, + const at::Tensor &attn_weight, + const int im2col_step) +{ + if (value.type().is_cuda()) + { +#ifdef WITH_CUDA + return ms_deform_attn_cuda_forward( + value, spatial_shapes, level_start_index, sampling_loc, attn_weight, im2col_step); +#else + AT_ERROR("Not compiled with GPU support"); +#endif + } + AT_ERROR("Not implemented on the CPU"); +} + +std::vector +ms_deform_attn_backward( + const at::Tensor &value, + const at::Tensor &spatial_shapes, + const at::Tensor &level_start_index, + const at::Tensor &sampling_loc, + const at::Tensor &attn_weight, + const at::Tensor &grad_output, + const int im2col_step) +{ + if (value.type().is_cuda()) + { +#ifdef WITH_CUDA + return ms_deform_attn_cuda_backward( + value, spatial_shapes, level_start_index, sampling_loc, attn_weight, grad_output, im2col_step); +#else + AT_ERROR("Not compiled with GPU support"); +#endif + } + AT_ERROR("Not implemented on the CPU"); +} + diff --git a/models/GroundingDINO/ops/src/vision.cpp b/models/GroundingDINO/ops/src/vision.cpp new file mode 100644 index 0000000000000000000000000000000000000000..2201f63a51dca16d0b31148ed2c9e8e47ec15bdc --- /dev/null +++ b/models/GroundingDINO/ops/src/vision.cpp @@ -0,0 +1,16 @@ +/*! +************************************************************************************************** +* Deformable DETR +* Copyright (c) 2020 SenseTime. All Rights Reserved. +* Licensed under the Apache License, Version 2.0 [see LICENSE for details] +************************************************************************************************** +* Modified from https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 +************************************************************************************************** +*/ + +#include "ms_deform_attn.h" + +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { + m.def("ms_deform_attn_forward", &ms_deform_attn_forward, "ms_deform_attn_forward"); + m.def("ms_deform_attn_backward", &ms_deform_attn_backward, "ms_deform_attn_backward"); +} diff --git a/models/GroundingDINO/ops/test.py b/models/GroundingDINO/ops/test.py new file mode 100644 index 0000000000000000000000000000000000000000..c3a1b918492835d45d19741b49754119047e5c5c --- /dev/null +++ b/models/GroundingDINO/ops/test.py @@ -0,0 +1,86 @@ +# ------------------------------------------------------------------------------------------------ +# Deformable DETR +# Copyright (c) 2020 SenseTime. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------------------------------ +# Modified from https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 +# ------------------------------------------------------------------------------------------------ + +from __future__ import absolute_import +from __future__ import print_function +from __future__ import division + +import time +import torch +import torch.nn as nn +from torch.autograd import gradcheck + +from functions.ms_deform_attn_func import MSDeformAttnFunction, ms_deform_attn_core_pytorch + + +N, M, D = 1, 2, 2 +Lq, L, P = 2, 2, 2 +shapes = torch.as_tensor([(6, 4), (3, 2)], dtype=torch.long).cuda() +level_start_index = torch.cat((shapes.new_zeros((1, )), shapes.prod(1).cumsum(0)[:-1])) +S = sum([(H*W).item() for H, W in shapes]) + + +torch.manual_seed(3) + + +@torch.no_grad() +def check_forward_equal_with_pytorch_double(): + value = torch.rand(N, S, M, D).cuda() * 0.01 + sampling_locations = torch.rand(N, Lq, M, L, P, 2).cuda() + attention_weights = torch.rand(N, Lq, M, L, P).cuda() + 1e-5 + attention_weights /= attention_weights.sum(-1, keepdim=True).sum(-2, keepdim=True) + im2col_step = 2 + output_pytorch = ms_deform_attn_core_pytorch(value.double(), shapes, sampling_locations.double(), attention_weights.double()).detach().cpu() + output_cuda = MSDeformAttnFunction.apply(value.double(), shapes, level_start_index, sampling_locations.double(), attention_weights.double(), im2col_step).detach().cpu() + fwdok = torch.allclose(output_cuda, output_pytorch) + max_abs_err = (output_cuda - output_pytorch).abs().max() + max_rel_err = ((output_cuda - output_pytorch).abs() / output_pytorch.abs()).max() + + print(f'* {fwdok} check_forward_equal_with_pytorch_double: max_abs_err {max_abs_err:.2e} max_rel_err {max_rel_err:.2e}') + + +@torch.no_grad() +def check_forward_equal_with_pytorch_float(): + value = torch.rand(N, S, M, D).cuda() * 0.01 + sampling_locations = torch.rand(N, Lq, M, L, P, 2).cuda() + attention_weights = torch.rand(N, Lq, M, L, P).cuda() + 1e-5 + attention_weights /= attention_weights.sum(-1, keepdim=True).sum(-2, keepdim=True) + im2col_step = 2 + output_pytorch = ms_deform_attn_core_pytorch(value, shapes, sampling_locations, attention_weights).detach().cpu() + output_cuda = MSDeformAttnFunction.apply(value, shapes, level_start_index, sampling_locations, attention_weights, im2col_step).detach().cpu() + fwdok = torch.allclose(output_cuda, output_pytorch, rtol=1e-2, atol=1e-3) + max_abs_err = (output_cuda - output_pytorch).abs().max() + max_rel_err = ((output_cuda - output_pytorch).abs() / output_pytorch.abs()).max() + + print(f'* {fwdok} check_forward_equal_with_pytorch_float: max_abs_err {max_abs_err:.2e} max_rel_err {max_rel_err:.2e}') + + +def check_gradient_numerical(channels=4, grad_value=True, grad_sampling_loc=True, grad_attn_weight=True): + + value = torch.rand(N, S, M, channels).cuda() * 0.01 + sampling_locations = torch.rand(N, Lq, M, L, P, 2).cuda() + attention_weights = torch.rand(N, Lq, M, L, P).cuda() + 1e-5 + attention_weights /= attention_weights.sum(-1, keepdim=True).sum(-2, keepdim=True) + im2col_step = 2 + func = MSDeformAttnFunction.apply + + value.requires_grad = grad_value + sampling_locations.requires_grad = grad_sampling_loc + attention_weights.requires_grad = grad_attn_weight + + gradok = gradcheck(func, (value.double(), shapes, level_start_index, sampling_locations.double(), attention_weights.double(), im2col_step)) + + print(f'* {gradok} check_gradient_numerical(D={channels})') + + +if __name__ == '__main__': + check_forward_equal_with_pytorch_double() + check_forward_equal_with_pytorch_float() + + for channels in [30, 32, 64, 71]: + check_gradient_numerical(channels, True, True, True) diff --git a/models/GroundingDINO/positional_encoding_loca.py b/models/GroundingDINO/positional_encoding_loca.py new file mode 100644 index 0000000000000000000000000000000000000000..9ca20f8e000c1f134b6d5108fe5ab626292a5519 --- /dev/null +++ b/models/GroundingDINO/positional_encoding_loca.py @@ -0,0 +1,30 @@ +import torch +from torch import nn + + +class PositionalEncodingsFixed(nn.Module): + + def __init__(self, emb_dim, temperature=10000): + + super(PositionalEncodingsFixed, self).__init__() + + self.emb_dim = emb_dim + self.temperature = temperature + + def _1d_pos_enc(self, mask, dim): + temp = torch.arange(self.emb_dim // 2).float().to(mask.device) + temp = self.temperature ** (2 * (temp.div(2, rounding_mode='floor')) / self.emb_dim) + + enc = (~mask).cumsum(dim).float().unsqueeze(-1) / temp + enc = torch.stack([ + enc[..., 0::2].sin(), enc[..., 1::2].cos() + ], dim=-1).flatten(-2) + + return enc + + def forward(self, bs, h, w, device): + mask = torch.zeros(bs, h, w, dtype=torch.bool, requires_grad=False, device=device) + x = self._1d_pos_enc(mask, dim=2) + y = self._1d_pos_enc(mask, dim=1) + + return torch.cat([y, x], dim=3).permute(0, 3, 1, 2) diff --git a/models/GroundingDINO/transformer.py b/models/GroundingDINO/transformer.py new file mode 100644 index 0000000000000000000000000000000000000000..1ec5c76fcd884dc1e9af1de0ffb6f2f619c8cc70 --- /dev/null +++ b/models/GroundingDINO/transformer.py @@ -0,0 +1,1003 @@ +# ------------------------------------------------------------------------ +# Grounding DINO +# url: https://github.com/IDEA-Research/GroundingDINO +# Copyright (c) 2023 IDEA. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------ +# DINO +# Copyright (c) 2022 IDEA. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------ +# Conditional DETR Transformer class. +# Copyright (c) 2021 Microsoft. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------ +# Modified from DETR (https://github.com/facebookresearch/detr) +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +# ------------------------------------------------------------------------ + +from typing import Optional + +import torch +import torch.utils.checkpoint as checkpoint +from torch import Tensor, nn + +from groundingdino.util.misc import inverse_sigmoid + +from .fuse_modules import BiAttentionBlock +from .ms_deform_attn import MultiScaleDeformableAttention as MSDeformAttn +from .transformer_vanilla import TransformerEncoderLayer +from .utils import ( + MLP, + _get_activation_fn, + _get_clones, + gen_encoder_output_proposals, + gen_sineembed_for_position, + get_sine_pos_embed, +) + + +class Transformer(nn.Module): + def __init__( + self, + d_model=256, + nhead=8, + num_queries=300, + num_encoder_layers=6, + num_unicoder_layers=0, + num_decoder_layers=6, + dim_feedforward=2048, + dropout=0.0, + activation="relu", + normalize_before=False, + return_intermediate_dec=False, + query_dim=4, + num_patterns=0, + # for deformable encoder + num_feature_levels=1, + enc_n_points=4, + dec_n_points=4, + # init query + learnable_tgt_init=False, + # two stage + two_stage_type="no", # ['no', 'standard', 'early', 'combine', 'enceachlayer', 'enclayer1'] + embed_init_tgt=False, + # for text + use_text_enhancer=False, + use_fusion_layer=False, + use_checkpoint=False, + use_transformer_ckpt=False, + use_text_cross_attention=False, + text_dropout=0.1, + fusion_dropout=0.1, + fusion_droppath=0.0, + ): + super().__init__() + self.num_feature_levels = num_feature_levels + self.num_encoder_layers = num_encoder_layers + self.num_unicoder_layers = num_unicoder_layers + self.num_decoder_layers = num_decoder_layers + self.num_queries = num_queries + assert query_dim == 4 + + # choose encoder layer type + encoder_layer = DeformableTransformerEncoderLayer( + d_model, + dim_feedforward, + dropout, + activation, + num_feature_levels, + nhead, + enc_n_points, + ) + + if use_text_enhancer: + text_enhance_layer = TransformerEncoderLayer( + d_model=d_model, + nhead=nhead // 2, + dim_feedforward=dim_feedforward // 2, + dropout=text_dropout, + ) + else: + text_enhance_layer = None + + if use_fusion_layer: + feature_fusion_layer = BiAttentionBlock( + v_dim=d_model, + l_dim=d_model, + embed_dim=dim_feedforward // 2, + num_heads=nhead // 2, + dropout=fusion_dropout, + drop_path=fusion_droppath, + ) + else: + feature_fusion_layer = None + + encoder_norm = nn.LayerNorm(d_model) if normalize_before else None + assert encoder_norm is None + self.encoder = TransformerEncoder( + encoder_layer, + num_encoder_layers, + d_model=d_model, + num_queries=num_queries, + text_enhance_layer=text_enhance_layer, + feature_fusion_layer=feature_fusion_layer, + use_checkpoint=use_checkpoint, + use_transformer_ckpt=use_transformer_ckpt, + ) + + # choose decoder layer type + decoder_layer = DeformableTransformerDecoderLayer( + d_model, + dim_feedforward, + dropout, + activation, + num_feature_levels, + nhead, + dec_n_points, + use_text_cross_attention=use_text_cross_attention, + ) + + decoder_norm = nn.LayerNorm(d_model) + self.decoder = TransformerDecoder( + decoder_layer, + num_decoder_layers, + decoder_norm, + return_intermediate=return_intermediate_dec, + d_model=d_model, + query_dim=query_dim, + num_feature_levels=num_feature_levels, + ) + + self.d_model = d_model + self.nhead = nhead + self.dec_layers = num_decoder_layers + self.num_queries = num_queries # useful for single stage model only + self.num_patterns = num_patterns + if not isinstance(num_patterns, int): + Warning("num_patterns should be int but {}".format(type(num_patterns))) + self.num_patterns = 0 + + if num_feature_levels > 1: + if self.num_encoder_layers > 0: + self.level_embed = nn.Parameter( + torch.Tensor(num_feature_levels, d_model) + ) + else: + self.level_embed = None + + self.learnable_tgt_init = learnable_tgt_init + assert learnable_tgt_init, "why not learnable_tgt_init" + self.embed_init_tgt = embed_init_tgt + if (two_stage_type != "no" and embed_init_tgt) or (two_stage_type == "no"): + self.tgt_embed = nn.Embedding(self.num_queries, d_model) + nn.init.normal_(self.tgt_embed.weight.data) + else: + self.tgt_embed = None + + # for two stage + self.two_stage_type = two_stage_type + assert two_stage_type in [ + "no", + "standard", + ], "unknown param {} of two_stage_type".format(two_stage_type) + if two_stage_type == "standard": + # anchor selection at the output of encoder + self.enc_output = nn.Linear(d_model, d_model) + self.enc_output_norm = nn.LayerNorm(d_model) + self.two_stage_wh_embedding = None + + if two_stage_type == "no": + self.init_ref_points(num_queries) # init self.refpoint_embed + + self.enc_out_class_embed = None + self.enc_out_bbox_embed = None + + self._reset_parameters() + + def _reset_parameters(self): + for p in self.parameters(): + if p.dim() > 1: + nn.init.xavier_uniform_(p) + for m in self.modules(): + if isinstance(m, MSDeformAttn): + m._reset_parameters() + if self.num_feature_levels > 1 and self.level_embed is not None: + nn.init.normal_(self.level_embed) + + def get_valid_ratio(self, mask): + _, H, W = mask.shape + valid_H = torch.sum(~mask[:, :, 0], 1) + valid_W = torch.sum(~mask[:, 0, :], 1) + valid_ratio_h = valid_H.float() / H + valid_ratio_w = valid_W.float() / W + valid_ratio = torch.stack([valid_ratio_w, valid_ratio_h], -1) + return valid_ratio + + def init_ref_points(self, use_num_queries): + self.refpoint_embed = nn.Embedding(use_num_queries, 4) + + def forward( + self, + srcs, + masks, + refpoint_embed, + pos_embeds, + tgt, + attn_mask=None, + text_dict=None, + ): + """ + Input: + - srcs: List of multi features [bs, ci, hi, wi] + - masks: List of multi masks [bs, hi, wi] + - refpoint_embed: [bs, num_dn, 4]. None in infer + - pos_embeds: List of multi pos embeds [bs, ci, hi, wi] + - tgt: [bs, num_dn, d_model]. None in infer + + """ + # prepare input for encoder + src_flatten = [] + mask_flatten = [] + lvl_pos_embed_flatten = [] + spatial_shapes = [] + for lvl, (src, mask, pos_embed) in enumerate(zip(srcs, masks, pos_embeds)): + bs, c, h, w = src.shape + spatial_shape = (h, w) + spatial_shapes.append(spatial_shape) + + src = src.flatten(2).transpose(1, 2) # bs, hw, c + mask = mask.flatten(1) # bs, hw + pos_embed = pos_embed.flatten(2).transpose(1, 2) # bs, hw, c + if self.num_feature_levels > 1 and self.level_embed is not None: + lvl_pos_embed = pos_embed + self.level_embed[lvl].view(1, 1, -1) + else: + lvl_pos_embed = pos_embed + lvl_pos_embed_flatten.append(lvl_pos_embed) + src_flatten.append(src) + mask_flatten.append(mask) + src_flatten = torch.cat(src_flatten, 1) # bs, \sum{hxw}, c + mask_flatten = torch.cat(mask_flatten, 1) # bs, \sum{hxw} + lvl_pos_embed_flatten = torch.cat(lvl_pos_embed_flatten, 1) # bs, \sum{hxw}, c + spatial_shapes = torch.as_tensor( + spatial_shapes, dtype=torch.long, device=src_flatten.device + ) + level_start_index = torch.cat( + (spatial_shapes.new_zeros((1,)), spatial_shapes.prod(1).cumsum(0)[:-1]) + ) + valid_ratios = torch.stack([self.get_valid_ratio(m) for m in masks], 1) + + # two stage + enc_topk_proposals = enc_refpoint_embed = None + + ######################################################### + # Begin Encoder + ######################################################### + + memory, memory_text = self.encoder( + src_flatten, + pos=lvl_pos_embed_flatten, + level_start_index=level_start_index, + spatial_shapes=spatial_shapes, + valid_ratios=valid_ratios, + key_padding_mask=mask_flatten, + memory_text=text_dict["encoded_text"], + text_attention_mask=~text_dict["text_token_mask"], + # we ~ the mask . False means use the token; True means pad the token + position_ids=text_dict["position_ids"], + text_self_attention_masks=text_dict["text_self_attention_masks"], + ) + + ######################################################### + # End Encoder + # - memory: bs, \sum{hw}, c + # - mask_flatten: bs, \sum{hw} + # - lvl_pos_embed_flatten: bs, \sum{hw}, c + # - enc_intermediate_output: None or (nenc+1, bs, nq, c) or (nenc, bs, nq, c) + # - enc_intermediate_refpoints: None or (nenc+1, bs, nq, c) or (nenc, bs, nq, c) + ######################################################### + text_dict["encoded_text"] = memory_text + # if os.environ.get("SHILONG_AMP_INFNAN_DEBUG") == '1': + # if memory.isnan().any() | memory.isinf().any(): + # import ipdb; ipdb.set_trace() + + if self.two_stage_type == "standard": # 把encoder的输出作为proposal + output_memory, output_proposals = gen_encoder_output_proposals( + memory, mask_flatten, spatial_shapes + ) + output_memory = self.enc_output_norm(self.enc_output(output_memory)) + + if text_dict is not None: + enc_outputs_class_unselected = self.enc_out_class_embed( + output_memory, text_dict + ) + else: + enc_outputs_class_unselected = self.enc_out_class_embed(output_memory) + + topk_logits = enc_outputs_class_unselected.max(-1)[0] + enc_outputs_coord_unselected = ( + self.enc_out_bbox_embed(output_memory) + output_proposals + ) # (bs, \sum{hw}, 4) unsigmoid + topk = self.num_queries + + topk_proposals = torch.topk(topk_logits, topk, dim=1)[1] # bs, nq + + # gather boxes + refpoint_embed_undetach = torch.gather( + enc_outputs_coord_unselected, + 1, + topk_proposals.unsqueeze(-1).repeat(1, 1, 4), + ) # unsigmoid + refpoint_embed_ = refpoint_embed_undetach.detach() + init_box_proposal = torch.gather( + output_proposals, 1, topk_proposals.unsqueeze(-1).repeat(1, 1, 4) + ).sigmoid() # sigmoid + + # gather tgt + tgt_undetach = torch.gather( + output_memory, + 1, + topk_proposals.unsqueeze(-1).repeat(1, 1, self.d_model), + ) + if self.embed_init_tgt: + tgt_ = ( + self.tgt_embed.weight[:, None, :].repeat(1, bs, 1).transpose(0, 1) + ) # nq, bs, d_model + else: + tgt_ = tgt_undetach.detach() + + if refpoint_embed is not None: + refpoint_embed = torch.cat([refpoint_embed, refpoint_embed_], dim=1) + tgt = torch.cat([tgt, tgt_], dim=1) + else: + refpoint_embed, tgt = refpoint_embed_, tgt_ + + elif self.two_stage_type == "no": + tgt_ = ( + self.tgt_embed.weight[:, None, :].repeat(1, bs, 1).transpose(0, 1) + ) # nq, bs, d_model + refpoint_embed_ = ( + self.refpoint_embed.weight[:, None, :].repeat(1, bs, 1).transpose(0, 1) + ) # nq, bs, 4 + + if refpoint_embed is not None: + refpoint_embed = torch.cat([refpoint_embed, refpoint_embed_], dim=1) + tgt = torch.cat([tgt, tgt_], dim=1) + else: + refpoint_embed, tgt = refpoint_embed_, tgt_ + + if self.num_patterns > 0: + tgt_embed = tgt.repeat(1, self.num_patterns, 1) + refpoint_embed = refpoint_embed.repeat(1, self.num_patterns, 1) + tgt_pat = self.patterns.weight[None, :, :].repeat_interleave( + self.num_queries, 1 + ) # 1, n_q*n_pat, d_model + tgt = tgt_embed + tgt_pat + + init_box_proposal = refpoint_embed_.sigmoid() + + else: + raise NotImplementedError( + "unknown two_stage_type {}".format(self.two_stage_type) + ) + ######################################################### + # End preparing tgt + # - tgt: bs, NQ, d_model + # - refpoint_embed(unsigmoid): bs, NQ, d_model + ######################################################### + + ######################################################### + # Begin Decoder + ######################################################### + + # memory torch.Size([2, 16320, 256]) + + # import pdb;pdb.set_trace() + hs, references = self.decoder( + tgt=tgt.transpose(0, 1), + memory=memory.transpose(0, 1), + memory_key_padding_mask=mask_flatten, + pos=lvl_pos_embed_flatten.transpose(0, 1), + refpoints_unsigmoid=refpoint_embed.transpose(0, 1), + level_start_index=level_start_index, + spatial_shapes=spatial_shapes, + valid_ratios=valid_ratios, + tgt_mask=attn_mask, + memory_text=text_dict["encoded_text"], + text_attention_mask=~text_dict["text_token_mask"], + # we ~ the mask . False means use the token; True means pad the token + ) + ######################################################### + # End Decoder + # hs: n_dec, bs, nq, d_model + # references: n_dec+1, bs, nq, query_dim + ######################################################### + + ######################################################### + # Begin postprocess + ######################################################### + if self.two_stage_type == "standard": + hs_enc = tgt_undetach.unsqueeze(0) + ref_enc = refpoint_embed_undetach.sigmoid().unsqueeze(0) + else: + hs_enc = ref_enc = None + ######################################################### + # End postprocess + # hs_enc: (n_enc+1, bs, nq, d_model) or (1, bs, nq, d_model) or (n_enc, bs, nq, d_model) or None + # ref_enc: (n_enc+1, bs, nq, query_dim) or (1, bs, nq, query_dim) or (n_enc, bs, nq, d_model) or None + ######################################################### + + return hs, references, hs_enc, ref_enc, init_box_proposal + # hs: (n_dec, bs, nq, d_model) + # references: sigmoid coordinates. (n_dec+1, bs, bq, 4) + # hs_enc: (n_enc+1, bs, nq, d_model) or (1, bs, nq, d_model) or None + # ref_enc: sigmoid coordinates. \ + # (n_enc+1, bs, nq, query_dim) or (1, bs, nq, query_dim) or None + + +class TransformerEncoder(nn.Module): + def __init__( + self, + encoder_layer, + num_layers, + d_model=256, + num_queries=300, + enc_layer_share=False, + text_enhance_layer=None, + feature_fusion_layer=None, + use_checkpoint=False, + use_transformer_ckpt=False, + ): + """_summary_ + + Args: + encoder_layer (_type_): _description_ + num_layers (_type_): _description_ + norm (_type_, optional): _description_. Defaults to None. + d_model (int, optional): _description_. Defaults to 256. + num_queries (int, optional): _description_. Defaults to 300. + enc_layer_share (bool, optional): _description_. Defaults to False. + + """ + super().__init__() + # prepare layers + self.layers = [] + self.text_layers = [] + self.fusion_layers = [] + if num_layers > 0: + self.layers = _get_clones( + encoder_layer, num_layers, layer_share=enc_layer_share + ) + + if text_enhance_layer is not None: + self.text_layers = _get_clones( + text_enhance_layer, num_layers, layer_share=enc_layer_share + ) + if feature_fusion_layer is not None: + self.fusion_layers = _get_clones( + feature_fusion_layer, num_layers, layer_share=enc_layer_share + ) + else: + self.layers = [] + del encoder_layer + + if text_enhance_layer is not None: + self.text_layers = [] + del text_enhance_layer + if feature_fusion_layer is not None: + self.fusion_layers = [] + del feature_fusion_layer + + self.query_scale = None + self.num_queries = num_queries + self.num_layers = num_layers + self.d_model = d_model + + self.use_checkpoint = use_checkpoint + self.use_transformer_ckpt = use_transformer_ckpt + + @staticmethod + def get_reference_points(spatial_shapes, valid_ratios, device): + reference_points_list = [] + for lvl, (H_, W_) in enumerate(spatial_shapes): + ref_y, ref_x = torch.meshgrid( + torch.linspace(0.5, H_ - 0.5, H_, dtype=torch.float32, device=device), + torch.linspace(0.5, W_ - 0.5, W_, dtype=torch.float32, device=device), + ) + ref_y = ref_y.reshape(-1)[None] / (valid_ratios[:, None, lvl, 1] * H_) + ref_x = ref_x.reshape(-1)[None] / (valid_ratios[:, None, lvl, 0] * W_) + ref = torch.stack((ref_x, ref_y), -1) + reference_points_list.append(ref) + reference_points = torch.cat(reference_points_list, 1) + reference_points = reference_points[:, :, None] * valid_ratios[:, None] + return reference_points + + def forward( + self, + # for images + src: Tensor, + pos: Tensor, + spatial_shapes: Tensor, + level_start_index: Tensor, + valid_ratios: Tensor, + key_padding_mask: Tensor, + # for texts + memory_text: Tensor = None, + text_attention_mask: Tensor = None, + pos_text: Tensor = None, + text_self_attention_masks: Tensor = None, + position_ids: Tensor = None, + ): + """ + Input: + - src: [bs, sum(hi*wi), 256] + - pos: pos embed for src. [bs, sum(hi*wi), 256] + - spatial_shapes: h,w of each level [num_level, 2] + - level_start_index: [num_level] start point of level in sum(hi*wi). + - valid_ratios: [bs, num_level, 2] + - key_padding_mask: [bs, sum(hi*wi)] + + - memory_text: bs, n_text, 256 + - text_attention_mask: bs, n_text + False for no padding; True for padding + - pos_text: bs, n_text, 256 + + - position_ids: bs, n_text + Intermedia: + - reference_points: [bs, sum(hi*wi), num_level, 2] + Outpus: + - output: [bs, sum(hi*wi), 256] + """ + + output = src + + # preparation and reshape + if self.num_layers > 0: + reference_points = self.get_reference_points( + spatial_shapes, valid_ratios, device=src.device + ) + + if self.text_layers: + # generate pos_text + bs, n_text, text_dim = memory_text.shape + if pos_text is None and position_ids is None: + pos_text = ( + torch.arange(n_text, device=memory_text.device) + .float() + .unsqueeze(0) + .unsqueeze(-1) + .repeat(bs, 1, 1) + ) + pos_text = get_sine_pos_embed( + pos_text, num_pos_feats=256, exchange_xy=False + ) + if position_ids is not None: + pos_text = get_sine_pos_embed( + position_ids[..., None], num_pos_feats=256, exchange_xy=False + ) + + # main process + for layer_id, layer in enumerate(self.layers): + # if output.isnan().any() or memory_text.isnan().any(): + # if os.environ.get('IPDB_SHILONG_DEBUG', None) == 'INFO': + # import ipdb; ipdb.set_trace() + if self.fusion_layers: + if self.use_checkpoint: + output, memory_text = checkpoint.checkpoint( + self.fusion_layers[layer_id], + output, + memory_text, + key_padding_mask, + text_attention_mask, + ) + else: + output, memory_text = self.fusion_layers[layer_id]( + v=output, + l=memory_text, + attention_mask_v=key_padding_mask, + attention_mask_l=text_attention_mask, + ) + + if self.text_layers: + memory_text = self.text_layers[layer_id]( + src=memory_text.transpose(0, 1), + src_mask=~text_self_attention_masks, # note we use ~ for mask here + src_key_padding_mask=text_attention_mask, + pos=(pos_text.transpose(0, 1) if pos_text is not None else None), + ).transpose(0, 1) + + # main process + if self.use_transformer_ckpt: + output = checkpoint.checkpoint( + layer, + output, + pos, + reference_points, + spatial_shapes, + level_start_index, + key_padding_mask, + ) + else: + output = layer( + src=output, + pos=pos, + reference_points=reference_points, + spatial_shapes=spatial_shapes, + level_start_index=level_start_index, + key_padding_mask=key_padding_mask, + ) + + return output, memory_text + + +class TransformerDecoder(nn.Module): + def __init__( + self, + decoder_layer, + num_layers, + norm=None, + return_intermediate=False, + d_model=256, + query_dim=4, + num_feature_levels=1, + ): + super().__init__() + if num_layers > 0: + self.layers = _get_clones(decoder_layer, num_layers) + else: + self.layers = [] + self.num_layers = num_layers + self.norm = norm + self.return_intermediate = return_intermediate + assert return_intermediate, "support return_intermediate only" + self.query_dim = query_dim + assert query_dim in [2, 4], "query_dim should be 2/4 but {}".format(query_dim) + self.num_feature_levels = num_feature_levels + + self.ref_point_head = MLP(query_dim // 2 * d_model, d_model, d_model, 2) + self.query_pos_sine_scale = None + + self.query_scale = None + self.bbox_embed = None + self.class_embed = None + + self.d_model = d_model + + self.ref_anchor_head = None + + def forward( + self, + tgt, + memory, + tgt_mask: Optional[Tensor] = None, + memory_mask: Optional[Tensor] = None, + tgt_key_padding_mask: Optional[Tensor] = None, + memory_key_padding_mask: Optional[Tensor] = None, + pos: Optional[Tensor] = None, + refpoints_unsigmoid: Optional[Tensor] = None, # num_queries, bs, 2 + # for memory + level_start_index: Optional[Tensor] = None, # num_levels + spatial_shapes: Optional[Tensor] = None, # bs, num_levels, 2 + valid_ratios: Optional[Tensor] = None, + # for text + memory_text: Optional[Tensor] = None, + text_attention_mask: Optional[Tensor] = None, + ): + """ + Input: + - tgt: nq, bs, d_model + - memory: hw, bs, d_model + - pos: hw, bs, d_model + - refpoints_unsigmoid: nq, bs, 2/4 + - valid_ratios/spatial_shapes: bs, nlevel, 2 + """ + output = tgt + + intermediate = [] + reference_points = refpoints_unsigmoid.sigmoid() + ref_points = [reference_points] + + for layer_id, layer in enumerate(self.layers): + if reference_points.shape[-1] == 4: + reference_points_input = ( + reference_points[:, :, None] + * torch.cat([valid_ratios, valid_ratios], -1)[None, :] + ) # nq, bs, nlevel, 4 + else: + assert reference_points.shape[-1] == 2 + reference_points_input = ( + reference_points[:, :, None] * valid_ratios[None, :] + ) + query_sine_embed = gen_sineembed_for_position( + reference_points_input[:, :, 0, :] + ) # nq, bs, 256*2 + + # conditional query + raw_query_pos = self.ref_point_head(query_sine_embed) # nq, bs, 256 + pos_scale = self.query_scale(output) if self.query_scale is not None else 1 + query_pos = pos_scale * raw_query_pos + # if os.environ.get("SHILONG_AMP_INFNAN_DEBUG") == '1': + # if query_pos.isnan().any() | query_pos.isinf().any(): + # import ipdb; ipdb.set_trace() + + # main process + output = layer( + tgt=output, + tgt_query_pos=query_pos, + tgt_query_sine_embed=query_sine_embed, + tgt_key_padding_mask=tgt_key_padding_mask, + tgt_reference_points=reference_points_input, + memory_text=memory_text, + text_attention_mask=text_attention_mask, + memory=memory, + memory_key_padding_mask=memory_key_padding_mask, + memory_level_start_index=level_start_index, + memory_spatial_shapes=spatial_shapes, + memory_pos=pos, + self_attn_mask=tgt_mask, + cross_attn_mask=memory_mask, + ) + if output.isnan().any() | output.isinf().any(): + print(f"output layer_id {layer_id} is nan") + try: + num_nan = output.isnan().sum().item() + num_inf = output.isinf().sum().item() + print(f"num_nan {num_nan}, num_inf {num_inf}") + except Exception as e: + print(e) + # if os.environ.get("SHILONG_AMP_INFNAN_DEBUG") == '1': + # import ipdb; ipdb.set_trace() + + # iter update + if self.bbox_embed is not None: + # box_holder = self.bbox_embed(output) + # box_holder[..., :self.query_dim] += inverse_sigmoid(reference_points) + # new_reference_points = box_holder[..., :self.query_dim].sigmoid() + + reference_before_sigmoid = inverse_sigmoid(reference_points) + delta_unsig = self.bbox_embed[layer_id](output) + outputs_unsig = delta_unsig + reference_before_sigmoid + new_reference_points = outputs_unsig.sigmoid() + + reference_points = new_reference_points.detach() + # if layer_id != self.num_layers - 1: + ref_points.append(new_reference_points) + + intermediate.append(self.norm(output)) + + # import pdb;pdb.set_trace() + + return [ + [itm_out.transpose(0, 1) for itm_out in intermediate], + [itm_refpoint.transpose(0, 1) for itm_refpoint in ref_points], + ] + + +class DeformableTransformerEncoderLayer(nn.Module): + def __init__( + self, + d_model=256, + d_ffn=1024, + dropout=0.1, + activation="relu", + n_levels=4, + n_heads=8, + n_points=4, + ): + super().__init__() + + # self attention + self.self_attn = MSDeformAttn( + embed_dim=d_model, + num_levels=n_levels, + num_heads=n_heads, + num_points=n_points, + batch_first=True, + ) + self.dropout1 = nn.Dropout(dropout) + self.norm1 = nn.LayerNorm(d_model) + + # ffn + self.linear1 = nn.Linear(d_model, d_ffn) + self.activation = _get_activation_fn(activation, d_model=d_ffn) + self.dropout2 = nn.Dropout(dropout) + self.linear2 = nn.Linear(d_ffn, d_model) + self.dropout3 = nn.Dropout(dropout) + self.norm2 = nn.LayerNorm(d_model) + + @staticmethod + def with_pos_embed(tensor, pos): + return tensor if pos is None else tensor + pos + + def forward_ffn(self, src): + src2 = self.linear2(self.dropout2(self.activation(self.linear1(src)))) + src = src + self.dropout3(src2) + src = self.norm2(src) + return src + + def forward( + self, + src, + pos, + reference_points, + spatial_shapes, + level_start_index, + key_padding_mask=None, + ): + # self attention + # import ipdb; ipdb.set_trace() + src2 = self.self_attn( + query=self.with_pos_embed(src, pos), + reference_points=reference_points, + value=src, + spatial_shapes=spatial_shapes, + level_start_index=level_start_index, + key_padding_mask=key_padding_mask, + ) + src = src + self.dropout1(src2) + src = self.norm1(src) + + # ffn + src = self.forward_ffn(src) + + return src + + +class DeformableTransformerDecoderLayer(nn.Module): + def __init__( + self, + d_model=256, + d_ffn=1024, + dropout=0.1, + activation="relu", + n_levels=4, + n_heads=8, + n_points=4, + use_text_feat_guide=False, + use_text_cross_attention=False, + ): + super().__init__() + + # cross attention + self.cross_attn = MSDeformAttn( + embed_dim=d_model, + num_levels=n_levels, + num_heads=n_heads, + num_points=n_points, + batch_first=True, + ) + self.dropout1 = nn.Dropout(dropout) if dropout > 0 else nn.Identity() + self.norm1 = nn.LayerNorm(d_model) + + # cross attention text + if use_text_cross_attention: + self.ca_text = nn.MultiheadAttention(d_model, n_heads, dropout=dropout) + self.catext_dropout = nn.Dropout(dropout) if dropout > 0 else nn.Identity() + self.catext_norm = nn.LayerNorm(d_model) + + # self attention + self.self_attn = nn.MultiheadAttention(d_model, n_heads, dropout=dropout) + self.dropout2 = nn.Dropout(dropout) if dropout > 0 else nn.Identity() + self.norm2 = nn.LayerNorm(d_model) + + # ffn + self.linear1 = nn.Linear(d_model, d_ffn) + self.activation = _get_activation_fn(activation, d_model=d_ffn, batch_dim=1) + self.dropout3 = nn.Dropout(dropout) if dropout > 0 else nn.Identity() + self.linear2 = nn.Linear(d_ffn, d_model) + self.dropout4 = nn.Dropout(dropout) if dropout > 0 else nn.Identity() + self.norm3 = nn.LayerNorm(d_model) + + self.key_aware_proj = None + self.use_text_feat_guide = use_text_feat_guide + assert not use_text_feat_guide + self.use_text_cross_attention = use_text_cross_attention + + def rm_self_attn_modules(self): + self.self_attn = None + self.dropout2 = None + self.norm2 = None + + @staticmethod + def with_pos_embed(tensor, pos): + return tensor if pos is None else tensor + pos + + def forward_ffn(self, tgt): + with torch.cuda.amp.autocast(enabled=False): + tgt2 = self.linear2(self.dropout3(self.activation(self.linear1(tgt)))) + tgt = tgt + self.dropout4(tgt2) + tgt = self.norm3(tgt) + return tgt + + def forward( + self, + # for tgt + tgt: Optional[Tensor], # nq, bs, d_model + tgt_query_pos: Optional[Tensor] = None, # pos for query. MLP(Sine(pos)) + tgt_query_sine_embed: Optional[Tensor] = None, # pos for query. Sine(pos) + tgt_key_padding_mask: Optional[Tensor] = None, + tgt_reference_points: Optional[Tensor] = None, # nq, bs, 4 + memory_text: Optional[Tensor] = None, # bs, num_token, d_model + text_attention_mask: Optional[Tensor] = None, # bs, num_token + # for memory + memory: Optional[Tensor] = None, # hw, bs, d_model + memory_key_padding_mask: Optional[Tensor] = None, + memory_level_start_index: Optional[Tensor] = None, # num_levels + memory_spatial_shapes: Optional[Tensor] = None, # bs, num_levels, 2 + memory_pos: Optional[Tensor] = None, # pos for memory + # sa + self_attn_mask: Optional[Tensor] = None, # mask used for self-attention + cross_attn_mask: Optional[Tensor] = None, # mask used for cross-attention + ): + """ + Input: + - tgt/tgt_query_pos: nq, bs, d_model + - + """ + assert cross_attn_mask is None + + # self attention + if self.self_attn is not None: + # import ipdb; ipdb.set_trace() + q = k = self.with_pos_embed(tgt, tgt_query_pos) + tgt2 = self.self_attn(q, k, tgt, attn_mask=self_attn_mask)[0] + tgt = tgt + self.dropout2(tgt2) + tgt = self.norm2(tgt) + + if self.use_text_cross_attention: + tgt2 = self.ca_text( + self.with_pos_embed(tgt, tgt_query_pos), + memory_text.transpose(0, 1), + memory_text.transpose(0, 1), + key_padding_mask=text_attention_mask, + )[0] + tgt = tgt + self.catext_dropout(tgt2) + tgt = self.catext_norm(tgt) + + tgt2 = self.cross_attn( + query=self.with_pos_embed(tgt, tgt_query_pos).transpose(0, 1), + reference_points=tgt_reference_points.transpose(0, 1).contiguous(), + value=memory.transpose(0, 1), + spatial_shapes=memory_spatial_shapes, + level_start_index=memory_level_start_index, + key_padding_mask=memory_key_padding_mask, + ).transpose(0, 1) + tgt = tgt + self.dropout1(tgt2) + tgt = self.norm1(tgt) + + # ffn + tgt = self.forward_ffn(tgt) + + return tgt + + +def build_transformer(args): + return Transformer( + d_model=args.hidden_dim, + dropout=args.dropout, + nhead=args.nheads, + num_queries=args.num_queries, + dim_feedforward=args.dim_feedforward, + num_encoder_layers=args.enc_layers, + num_decoder_layers=args.dec_layers, + normalize_before=args.pre_norm, + return_intermediate_dec=True, + query_dim=args.query_dim, + activation=args.transformer_activation, + num_patterns=args.num_patterns, + num_feature_levels=args.num_feature_levels, + enc_n_points=args.enc_n_points, + dec_n_points=args.dec_n_points, + learnable_tgt_init=True, + # two stage + two_stage_type=args.two_stage_type, # ['no', 'standard', 'early'] + embed_init_tgt=args.embed_init_tgt, + use_text_enhancer=args.use_text_enhancer, + use_fusion_layer=args.use_fusion_layer, + use_checkpoint=args.use_checkpoint, + use_transformer_ckpt=args.use_transformer_ckpt, + use_text_cross_attention=args.use_text_cross_attention, + text_dropout=args.text_dropout, + fusion_dropout=args.fusion_dropout, + fusion_droppath=args.fusion_droppath, + ) diff --git a/models/GroundingDINO/transformer_loca.py b/models/GroundingDINO/transformer_loca.py new file mode 100644 index 0000000000000000000000000000000000000000..f4096b3665779238b661e8067ccd5342691d0077 --- /dev/null +++ b/models/GroundingDINO/transformer_loca.py @@ -0,0 +1,94 @@ +from .mlp_loca import MLP + +from torch import nn + + +class TransformerEncoder(nn.Module): + + def __init__( + self, + num_layers: int, + emb_dim: int, + num_heads: int, + dropout: float, + layer_norm_eps: float, + mlp_factor: int, + norm_first: bool, + activation: nn.Module, + norm: bool, + ): + + super(TransformerEncoder, self).__init__() + + self.layers = nn.ModuleList([ + TransformerEncoderLayer( + emb_dim, num_heads, dropout, layer_norm_eps, + mlp_factor, norm_first, activation + ) for _ in range(num_layers) + ]) + + self.norm = nn.LayerNorm(emb_dim, layer_norm_eps) if norm else nn.Identity() + + def forward(self, src, pos_emb, src_mask, src_key_padding_mask): + output = src + for layer in self.layers: + output = layer(output, pos_emb, src_mask, src_key_padding_mask) + return self.norm(output) + + +class TransformerEncoderLayer(nn.Module): + + def __init__( + self, + emb_dim: int, + num_heads: int, + dropout: float, + layer_norm_eps: float, + mlp_factor: int, + norm_first: bool, + activation: nn.Module, + ): + super(TransformerEncoderLayer, self).__init__() + + self.norm_first = norm_first + + self.norm1 = nn.LayerNorm(emb_dim, layer_norm_eps) + self.norm2 = nn.LayerNorm(emb_dim, layer_norm_eps) + self.dropout1 = nn.Dropout(dropout) + self.dropout2 = nn.Dropout(dropout) + + self.self_attn = nn.MultiheadAttention( + emb_dim, num_heads, dropout + ) + self.mlp = MLP(emb_dim, mlp_factor * emb_dim, dropout, activation) + + def with_emb(self, x, emb): + return x if emb is None else x + emb + + def forward(self, src, pos_emb, src_mask, src_key_padding_mask): + if self.norm_first: + src_norm = self.norm1(src) + q = k = src_norm + pos_emb + src = src + self.dropout1(self.self_attn( + query=q, + key=k, + value=src_norm, + attn_mask=src_mask, + key_padding_mask=src_key_padding_mask + )[0]) + + src_norm = self.norm2(src) + src = src + self.dropout2(self.mlp(src_norm)) + else: + q = k = src + pos_emb + src = self.norm1(src + self.dropout1(self.self_attn( + query=q, + key=k, + value=src, + attn_mask=src_mask, + key_padding_mask=src_key_padding_mask + )[0])) + + src = self.norm2(src + self.dropout2(self.mlp(src))) + + return src diff --git a/models/GroundingDINO/transformer_vanilla.py b/models/GroundingDINO/transformer_vanilla.py new file mode 100644 index 0000000000000000000000000000000000000000..10c0920c1a217af5bb3e1b13077568035ab3b7b5 --- /dev/null +++ b/models/GroundingDINO/transformer_vanilla.py @@ -0,0 +1,123 @@ +# ------------------------------------------------------------------------ +# Grounding DINO +# url: https://github.com/IDEA-Research/GroundingDINO +# Copyright (c) 2023 IDEA. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------ +# Copyright (c) Aishwarya Kamath & Nicolas Carion. Licensed under the Apache License 2.0. All Rights Reserved +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved +""" +DETR Transformer class. + +Copy-paste from torch.nn.Transformer with modifications: + * positional encodings are passed in MHattention + * extra LN at the end of encoder is removed + * decoder returns a stack of activations from all decoding layers +""" +from typing import Optional + +import torch +import torch.nn.functional as F +from torch import Tensor, nn + +from .utils import ( + MLP, + _get_activation_fn, + _get_clones, + gen_encoder_output_proposals, + gen_sineembed_for_position, + sigmoid_focal_loss, +) + + +class TextTransformer(nn.Module): + def __init__(self, num_layers, d_model=256, nheads=8, dim_feedforward=2048, dropout=0.1): + super().__init__() + self.num_layers = num_layers + self.d_model = d_model + self.nheads = nheads + self.dim_feedforward = dim_feedforward + self.norm = None + + single_encoder_layer = TransformerEncoderLayer( + d_model=d_model, nhead=nheads, dim_feedforward=dim_feedforward, dropout=dropout + ) + self.layers = _get_clones(single_encoder_layer, num_layers) + + def forward(self, memory_text: torch.Tensor, text_attention_mask: torch.Tensor): + """ + + Args: + text_attention_mask: bs, num_token + memory_text: bs, num_token, d_model + + Raises: + RuntimeError: _description_ + + Returns: + output: bs, num_token, d_model + """ + + output = memory_text.transpose(0, 1) + + for layer in self.layers: + output = layer(output, src_key_padding_mask=text_attention_mask) + + if self.norm is not None: + output = self.norm(output) + + return output.transpose(0, 1) + + +class TransformerEncoderLayer(nn.Module): + def __init__( + self, + d_model, + nhead, + dim_feedforward=2048, + dropout=0.1, + activation="relu", + normalize_before=False, + ): + super().__init__() + self.self_attn = nn.MultiheadAttention(d_model, nhead, dropout=dropout) + # Implementation of Feedforward model + self.linear1 = nn.Linear(d_model, dim_feedforward) + self.dropout = nn.Dropout(dropout) + self.linear2 = nn.Linear(dim_feedforward, d_model) + + self.norm1 = nn.LayerNorm(d_model) + self.norm2 = nn.LayerNorm(d_model) + self.dropout1 = nn.Dropout(dropout) + self.dropout2 = nn.Dropout(dropout) + + self.activation = _get_activation_fn(activation) + self.normalize_before = normalize_before + self.nhead = nhead + + def with_pos_embed(self, tensor, pos: Optional[Tensor]): + return tensor if pos is None else tensor + pos + + def forward( + self, + src, + src_mask: Optional[Tensor] = None, + src_key_padding_mask: Optional[Tensor] = None, + pos: Optional[Tensor] = None, + ): + # repeat attn mask + if src_mask.dim() == 3 and src_mask.shape[0] == src.shape[1]: + # bs, num_q, num_k + src_mask = src_mask.repeat(self.nhead, 1, 1) + + q = k = self.with_pos_embed(src, pos) + + src2 = self.self_attn(q, k, value=src, attn_mask=src_mask)[0] + + # src2 = self.self_attn(q, k, value=src, attn_mask=src_mask, key_padding_mask=src_key_padding_mask)[0] + src = src + self.dropout1(src2) + src = self.norm1(src) + src2 = self.linear2(self.dropout(self.activation(self.linear1(src)))) + src = src + self.dropout2(src2) + src = self.norm2(src) + return src diff --git a/models/GroundingDINO/utils.py b/models/GroundingDINO/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..3c8e4b194290f178b45c68183c4113ebc768df39 --- /dev/null +++ b/models/GroundingDINO/utils.py @@ -0,0 +1,273 @@ +# ------------------------------------------------------------------------ +# Grounding DINO +# url: https://github.com/IDEA-Research/GroundingDINO +# Copyright (c) 2023 IDEA. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------ + +import copy +import math + +import torch +import torch.nn.functional as F +from torch import Tensor, nn + + +def _get_clones(module, N, layer_share=False): + # import ipdb; ipdb.set_trace() + if layer_share: + return nn.ModuleList([module for i in range(N)]) + else: + return nn.ModuleList([copy.deepcopy(module) for i in range(N)]) + + +def get_sine_pos_embed( + pos_tensor: torch.Tensor, + num_pos_feats: int = 128, + temperature: int = 10000, + exchange_xy: bool = True, +): + """generate sine position embedding from a position tensor + Args: + pos_tensor (torch.Tensor): shape: [..., n]. + num_pos_feats (int): projected shape for each float in the tensor. + temperature (int): temperature in the sine/cosine function. + exchange_xy (bool, optional): exchange pos x and pos y. \ + For example, input tensor is [x,y], the results will be [pos(y), pos(x)]. Defaults to True. + Returns: + pos_embed (torch.Tensor): shape: [..., n*num_pos_feats]. + """ + scale = 2 * math.pi + dim_t = torch.arange(num_pos_feats, dtype=torch.float32, device=pos_tensor.device) + dim_t = temperature ** (2 * torch.div(dim_t, 2, rounding_mode="floor") / num_pos_feats) + + def sine_func(x: torch.Tensor): + sin_x = x * scale / dim_t + sin_x = torch.stack((sin_x[..., 0::2].sin(), sin_x[..., 1::2].cos()), dim=3).flatten(2) + return sin_x + + pos_res = [sine_func(x) for x in pos_tensor.split([1] * pos_tensor.shape[-1], dim=-1)] + if exchange_xy: + pos_res[0], pos_res[1] = pos_res[1], pos_res[0] + pos_res = torch.cat(pos_res, dim=-1) + return pos_res + + +def gen_encoder_output_proposals( + memory: Tensor, memory_padding_mask: Tensor, spatial_shapes: Tensor, learnedwh=None +): + """ + Input: + - memory: bs, \sum{hw}, d_model + - memory_padding_mask: bs, \sum{hw} + - spatial_shapes: nlevel, 2 + - learnedwh: 2 + Output: + - output_memory: bs, \sum{hw}, d_model + - output_proposals: bs, \sum{hw}, 4 + """ + N_, S_, C_ = memory.shape + proposals = [] + _cur = 0 + for lvl, (H_, W_) in enumerate(spatial_shapes): + mask_flatten_ = memory_padding_mask[:, _cur : (_cur + H_ * W_)].view(N_, H_, W_, 1) + valid_H = torch.sum(~mask_flatten_[:, :, 0, 0], 1) + valid_W = torch.sum(~mask_flatten_[:, 0, :, 0], 1) + + # import ipdb; ipdb.set_trace() + + grid_y, grid_x = torch.meshgrid( + torch.linspace(0, H_ - 1, H_, dtype=torch.float32, device=memory.device), + torch.linspace(0, W_ - 1, W_, dtype=torch.float32, device=memory.device), + ) + grid = torch.cat([grid_x.unsqueeze(-1), grid_y.unsqueeze(-1)], -1) # H_, W_, 2 + + scale = torch.cat([valid_W.unsqueeze(-1), valid_H.unsqueeze(-1)], 1).view(N_, 1, 1, 2) + grid = (grid.unsqueeze(0).expand(N_, -1, -1, -1) + 0.5) / scale + + if learnedwh is not None: + # import ipdb; ipdb.set_trace() + wh = torch.ones_like(grid) * learnedwh.sigmoid() * (2.0**lvl) + else: + wh = torch.ones_like(grid) * 0.05 * (2.0**lvl) + + # scale = torch.cat([W_[None].unsqueeze(-1), H_[None].unsqueeze(-1)], 1).view(1, 1, 1, 2).repeat(N_, 1, 1, 1) + # grid = (grid.unsqueeze(0).expand(N_, -1, -1, -1) + 0.5) / scale + # wh = torch.ones_like(grid) / scale + proposal = torch.cat((grid, wh), -1).view(N_, -1, 4) + proposals.append(proposal) + _cur += H_ * W_ + # import ipdb; ipdb.set_trace() + output_proposals = torch.cat(proposals, 1) + output_proposals_valid = ((output_proposals > 0.01) & (output_proposals < 0.99)).all( + -1, keepdim=True + ) + output_proposals = torch.log(output_proposals / (1 - output_proposals)) # unsigmoid + output_proposals = output_proposals.masked_fill(memory_padding_mask.unsqueeze(-1), float("inf")) + output_proposals = output_proposals.masked_fill(~output_proposals_valid, float("inf")) + + output_memory = memory + output_memory = output_memory.masked_fill(memory_padding_mask.unsqueeze(-1), float(0)) + output_memory = output_memory.masked_fill(~output_proposals_valid, float(0)) + + # output_memory = output_memory.masked_fill(memory_padding_mask.unsqueeze(-1), float('inf')) + # output_memory = output_memory.masked_fill(~output_proposals_valid, float('inf')) + + return output_memory, output_proposals + + +class RandomBoxPerturber: + def __init__( + self, x_noise_scale=0.2, y_noise_scale=0.2, w_noise_scale=0.2, h_noise_scale=0.2 + ) -> None: + self.noise_scale = torch.Tensor( + [x_noise_scale, y_noise_scale, w_noise_scale, h_noise_scale] + ) + + def __call__(self, refanchors: Tensor) -> Tensor: + nq, bs, query_dim = refanchors.shape + device = refanchors.device + + noise_raw = torch.rand_like(refanchors) + noise_scale = self.noise_scale.to(device)[:query_dim] + + new_refanchors = refanchors * (1 + (noise_raw - 0.5) * noise_scale) + return new_refanchors.clamp_(0, 1) + + +def sigmoid_focal_loss( + inputs, targets, num_boxes, alpha: float = 0.25, gamma: float = 2, no_reduction=False +): + """ + Loss used in RetinaNet for dense detection: https://arxiv.org/abs/1708.02002. + Args: + inputs: A float tensor of arbitrary shape. + The predictions for each example. + targets: A float tensor with the same shape as inputs. Stores the binary + classification label for each element in inputs + (0 for the negative class and 1 for the positive class). + alpha: (optional) Weighting factor in range (0,1) to balance + positive vs negative examples. Default = -1 (no weighting). + gamma: Exponent of the modulating factor (1 - p_t) to + balance easy vs hard examples. + Returns: + Loss tensor + """ + prob = inputs.sigmoid() + ce_loss = F.binary_cross_entropy_with_logits(inputs, targets, reduction="none") + p_t = prob * targets + (1 - prob) * (1 - targets) + loss = ce_loss * ((1 - p_t) ** gamma) + + if alpha >= 0: + alpha_t = alpha * targets + (1 - alpha) * (1 - targets) + loss = alpha_t * loss + + if no_reduction: + return loss + + return loss.mean(1).sum() / num_boxes + + +class MLP(nn.Module): + """Very simple multi-layer perceptron (also called FFN)""" + + def __init__(self, input_dim, hidden_dim, output_dim, num_layers): + super().__init__() + self.num_layers = num_layers + h = [hidden_dim] * (num_layers - 1) + self.layers = nn.ModuleList( + nn.Linear(n, k) for n, k in zip([input_dim] + h, h + [output_dim]) + ) + + def forward(self, x): + for i, layer in enumerate(self.layers): + x = F.relu(layer(x)) if i < self.num_layers - 1 else layer(x) + return x + + +def _get_activation_fn(activation, d_model=256, batch_dim=0): + """Return an activation function given a string""" + if activation == "relu": + return F.relu + if activation == "gelu": + return F.gelu + if activation == "glu": + return F.glu + if activation == "prelu": + return nn.PReLU() + if activation == "selu": + return F.selu + + raise RuntimeError(f"activation should be relu/gelu, not {activation}.") + + +def gen_sineembed_for_position(pos_tensor): + # n_query, bs, _ = pos_tensor.size() + # sineembed_tensor = torch.zeros(n_query, bs, 256) + scale = 2 * math.pi + dim_t = torch.arange(128, dtype=torch.float32, device=pos_tensor.device) + dim_t = 10000 ** (2 * (torch.div(dim_t, 2, rounding_mode='floor')) / 128) + x_embed = pos_tensor[:, :, 0] * scale + y_embed = pos_tensor[:, :, 1] * scale + pos_x = x_embed[:, :, None] / dim_t + pos_y = y_embed[:, :, None] / dim_t + pos_x = torch.stack((pos_x[:, :, 0::2].sin(), pos_x[:, :, 1::2].cos()), dim=3).flatten(2) + pos_y = torch.stack((pos_y[:, :, 0::2].sin(), pos_y[:, :, 1::2].cos()), dim=3).flatten(2) + if pos_tensor.size(-1) == 2: + pos = torch.cat((pos_y, pos_x), dim=2) + elif pos_tensor.size(-1) == 4: + w_embed = pos_tensor[:, :, 2] * scale + pos_w = w_embed[:, :, None] / dim_t + pos_w = torch.stack((pos_w[:, :, 0::2].sin(), pos_w[:, :, 1::2].cos()), dim=3).flatten(2) + + h_embed = pos_tensor[:, :, 3] * scale + pos_h = h_embed[:, :, None] / dim_t + pos_h = torch.stack((pos_h[:, :, 0::2].sin(), pos_h[:, :, 1::2].cos()), dim=3).flatten(2) + + pos = torch.cat((pos_y, pos_x, pos_w, pos_h), dim=2) + else: + raise ValueError("Unknown pos_tensor shape(-1):{}".format(pos_tensor.size(-1))) + return pos + + +class ContrastiveEmbed(nn.Module): + def __init__(self, max_text_len=256): + """ + Args: + max_text_len: max length of text. + """ + super().__init__() + self.max_text_len = max_text_len + + def forward(self, x, text_dict): + """_summary_ + + Args: + x (_type_): _description_ + text_dict (_type_): _description_ + { + 'encoded_text': encoded_text, # bs, 195, d_model + 'text_token_mask': text_token_mask, # bs, 195 + # True for used tokens. False for padding tokens + } + Returns: + _type_: _description_ + """ + assert isinstance(text_dict, dict) + # print(x) #torch.Size([2, 16320, 256]) + # print(text_dict) + + # import pdb;pdb.set_trace() + y = text_dict["encoded_text"] #torch.Size([2, 195, 256]) + text_token_mask = text_dict["text_token_mask"] + + res = x @ y.transpose(-1, -2) + res.masked_fill_(~text_token_mask[:, None, :], float("-inf")) + # 接着，对res进行掩码操作，将未使用的文本token（即padding的token）对应的得分置为负无穷float("-inf")。这是为了在计算相似度时，排除padding部分的影响。 + + + # padding to max_text_len + new_res = torch.full((*res.shape[:-1], self.max_text_len), float("-inf"), device=res.device) + new_res[..., : res.shape[-1]] = res #torch.Size([2, 16320, 195]) + + return new_res diff --git a/models/__init__.py b/models/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..8b123bbf7ac35014e68a70c2056f1325389f0ecc --- /dev/null +++ b/models/__init__.py @@ -0,0 +1,10 @@ +# ------------------------------------------------------------------------ +# DINO +# Copyright (c) 2022 IDEA. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved +from .GroundingDINO import build_groundingdino + +def build_model(args): + return build(args) diff --git a/models/__pycache__/__init__.cpython-38.pyc b/models/__pycache__/__init__.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8cd8047b0148d528f09cae07fef89bf624a62d52 Binary files /dev/null and b/models/__pycache__/__init__.cpython-38.pyc differ diff --git a/models/__pycache__/__init__.cpython-39.pyc b/models/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..af6524b128039a854c5e4755e756a23b17f9548e Binary files /dev/null and b/models/__pycache__/__init__.cpython-39.pyc differ diff --git a/models/__pycache__/registry.cpython-38.pyc b/models/__pycache__/registry.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7a5428b03e9d91be7032b2632c910c6f3b9a6085 Binary files /dev/null and b/models/__pycache__/registry.cpython-38.pyc differ diff --git a/models/__pycache__/registry.cpython-39.pyc b/models/__pycache__/registry.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4d2d859c4fe48008e6d6631d865fd0040b31e1b7 Binary files /dev/null and b/models/__pycache__/registry.cpython-39.pyc differ diff --git a/models/registry.py b/models/registry.py new file mode 100644 index 0000000000000000000000000000000000000000..f438c6e3918a84cc2004b5da9c1d79d18cfb3118 --- /dev/null +++ b/models/registry.py @@ -0,0 +1,58 @@ +# -*- coding: utf-8 -*- +# @Author: Yihao Chen +# @Date: 2021-08-16 16:03:17 +# @Last Modified by: Shilong Liu +# @Last Modified time: 2022-01-23 15:26 +# modified from mmcv + +import inspect +from functools import partial + + +class Registry(object): + + def __init__(self, name): + self._name = name + self._module_dict = dict() + + def __repr__(self): + format_str = self.__class__.__name__ + '(name={}, items={})'.format( + self._name, list(self._module_dict.keys())) + return format_str + + def __len__(self): + return len(self._module_dict) + + @property + def name(self): + return self._name + + @property + def module_dict(self): + return self._module_dict + + def get(self, key): + return self._module_dict.get(key, None) + + def registe_with_name(self, module_name=None, force=False): + return partial(self.register, module_name=module_name, force=force) + + def register(self, module_build_function, module_name=None, force=False): + """Register a module build function. + Args: + module (:obj:`nn.Module`): Module to be registered. + """ + if not inspect.isfunction(module_build_function): + raise TypeError('module_build_function must be a function, but got {}'.format( + type(module_build_function))) + if module_name is None: + module_name = module_build_function.__name__ + if not force and module_name in self._module_dict: + raise KeyError('{} is already registered in {}'.format( + module_name, self.name)) + self._module_dict[module_name] = module_build_function + + return module_build_function + +MODULE_BUILD_FUNCS = Registry('model build functions') + diff --git a/nohup.out b/nohup.out new file mode 100644 index 0000000000000000000000000000000000000000..abba2e24480b2cdf84952da918954b55bf08156b --- /dev/null +++ b/nohup.out @@ -0,0 +1,398 @@ +/home/niki/anaconda3/envs/gradio/lib/python3.9/site-packages/torch/functional.py:507: UserWarning: torch.meshgrid: in an upcoming release, it will be required to pass the indexing argument. (Triggered internally at ../aten/src/ATen/native/TensorShape.cpp:3549.) + return _VF.meshgrid(tensors, **kwargs) # type: ignore[attr-defined] +Some weights of BertModel were not initialized from the model checkpoint at checkpoints/bert-base-uncased and are newly initialized: ['pooler.dense.bias', 'pooler.dense.weight'] +You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference. +huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks... +To disable this warning, you can either: + - Avoid using `tokenizers` before the fork if possible + - Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false) +/home/niki/anaconda3/envs/gradio/lib/python3.9/site-packages/torch/functional.py:507: UserWarning: torch.meshgrid: in an upcoming release, it will be required to pass the indexing argument. (Triggered internally at ../aten/src/ATen/native/TensorShape.cpp:3549.) + return _VF.meshgrid(tensors, **kwargs) # type: ignore[attr-defined] +Some weights of BertModel were not initialized from the model checkpoint at checkpoints/bert-base-uncased and are newly initialized: ['pooler.dense.bias', 'pooler.dense.weight'] +You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference. +final text_encoder_type: checkpoints/bert-base-uncased +load tokenizer done. +final text_encoder_type: checkpoints/bert-base-uncased +load tokenizer done. +Running on local URL: http://127.0.0.1:7860 +Traceback (most recent call last): + File "/home/niki/gradio-tutorial/app.py", line 230, in + demo.launch(share=True, allowed_paths=['teaser-gradio.jpg']) + File "/home/niki/anaconda3/envs/gradio/lib/python3.9/site-packages/gradio/blocks.py", line 2441, in launch + share_url = networking.setup_tunnel( + File "/home/niki/anaconda3/envs/gradio/lib/python3.9/site-packages/gradio/networking.py", line 31, in setup_tunnel + response = httpx.get(GRADIO_API_SERVER, timeout=30) + File "/home/niki/anaconda3/envs/gradio/lib/python3.9/site-packages/httpx/_api.py", line 198, in get + return request( + File "/home/niki/anaconda3/envs/gradio/lib/python3.9/site-packages/httpx/_api.py", line 106, in request + return client.request( + File "/home/niki/anaconda3/envs/gradio/lib/python3.9/site-packages/httpx/_client.py", line 827, in request + return self.send(request, auth=auth, follow_redirects=follow_redirects) + File "/home/niki/anaconda3/envs/gradio/lib/python3.9/site-packages/httpx/_client.py", line 914, in send + response = self._send_handling_auth( + File "/home/niki/anaconda3/envs/gradio/lib/python3.9/site-packages/httpx/_client.py", line 942, in _send_handling_auth + response = self._send_handling_redirects( + File "/home/niki/anaconda3/envs/gradio/lib/python3.9/site-packages/httpx/_client.py", line 979, in _send_handling_redirects + response = self._send_single_request(request) + File "/home/niki/anaconda3/envs/gradio/lib/python3.9/site-packages/httpx/_client.py", line 1015, in _send_single_request + response = transport.handle_request(request) + File "/home/niki/anaconda3/envs/gradio/lib/python3.9/site-packages/httpx/_transports/default.py", line 233, in handle_request + resp = self._pool.handle_request(req) + File "/home/niki/anaconda3/envs/gradio/lib/python3.9/site-packages/httpcore/_sync/connection_pool.py", line 216, in handle_request + raise exc from None + File "/home/niki/anaconda3/envs/gradio/lib/python3.9/site-packages/httpcore/_sync/connection_pool.py", line 196, in handle_request + response = connection.handle_request( + File "/home/niki/anaconda3/envs/gradio/lib/python3.9/site-packages/httpcore/_sync/connection.py", line 99, in handle_request + raise exc + File "/home/niki/anaconda3/envs/gradio/lib/python3.9/site-packages/httpcore/_sync/connection.py", line 76, in handle_request + stream = self._connect(request) + File "/home/niki/anaconda3/envs/gradio/lib/python3.9/site-packages/httpcore/_sync/connection.py", line 154, in _connect + stream = stream.start_tls(**kwargs) + File "/home/niki/anaconda3/envs/gradio/lib/python3.9/site-packages/httpcore/_backends/sync.py", line 163, in start_tls + sock = ssl_context.wrap_socket( + File "/home/niki/anaconda3/envs/gradio/lib/python3.9/ssl.py", line 501, in wrap_socket + return self.sslsocket_class._create( + File "/home/niki/anaconda3/envs/gradio/lib/python3.9/ssl.py", line 1074, in _create + self.do_handshake() + File "/home/niki/anaconda3/envs/gradio/lib/python3.9/ssl.py", line 1343, in do_handshake + self._sslobj.do_handshake() +KeyboardInterrupt +/home/niki/anaconda3/envs/gradio/lib/python3.9/site-packages/torch/functional.py:507: UserWarning: torch.meshgrid: in an upcoming release, it will be required to pass the indexing argument. (Triggered internally at ../aten/src/ATen/native/TensorShape.cpp:3549.) + return _VF.meshgrid(tensors, **kwargs) # type: ignore[attr-defined] +final text_encoder_type: checkpoints/bert-base-uncased +load tokenizer done. +Some weights of BertModel were not initialized from the model checkpoint at checkpoints/bert-base-uncased and are newly initialized: ['pooler.dense.bias', 'pooler.dense.weight'] +You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference. +final text_encoder_type: checkpoints/bert-base-uncased +load tokenizer done. +Running on local URL: http://127.0.0.1:7860 +huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks... +To disable this warning, you can either: + - Avoid using `tokenizers` before the fork if possible + - Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false) +Running on public URL: https://ffc289d2866c212c8b.gradio.live + +This share link expires in 72 hours. For free permanent hosting and GPU upgrades, run `gradio deploy` from Terminal to deploy to Spaces (https://huggingface.co/spaces) +[] +/home/niki/anaconda3/envs/gradio/lib/python3.9/site-packages/transformers/modeling_utils.py:977: FutureWarning: The `device` argument is deprecated and will be removed in v5 of Transformers. + warnings.warn( +/home/niki/anaconda3/envs/gradio/lib/python3.9/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/home/niki/anaconda3/envs/gradio/lib/python3.9/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None + warnings.warn( +[tensor([], device='cuda:0')] +0 +['[CLS]', 'cat', '.', '[SEP]'] +[0, 1, 2, 3] +torch.Size([900, 256]) +torch.Size([900, 4]) +[] +/home/niki/anaconda3/envs/gradio/lib/python3.9/site-packages/transformers/modeling_utils.py:977: FutureWarning: The `device` argument is deprecated and will be removed in v5 of Transformers. + warnings.warn( +/home/niki/anaconda3/envs/gradio/lib/python3.9/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/home/niki/anaconda3/envs/gradio/lib/python3.9/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None + warnings.warn( +[tensor([], device='cuda:0')] +0 +['[CLS]', 'sign', '.', '[SEP]'] +[0, 1, 2, 3] +torch.Size([900, 256]) +torch.Size([900, 4]) +[] +/home/niki/anaconda3/envs/gradio/lib/python3.9/site-packages/transformers/modeling_utils.py:977: FutureWarning: The `device` argument is deprecated and will be removed in v5 of Transformers. + warnings.warn( +/home/niki/anaconda3/envs/gradio/lib/python3.9/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/home/niki/anaconda3/envs/gradio/lib/python3.9/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None + warnings.warn( +[tensor([], device='cuda:0')] +0 +['[CLS]', 'cushion', '.', '[SEP]'] +[0, 1, 2, 3] +torch.Size([900, 256]) +torch.Size([900, 4]) +[] +/home/niki/anaconda3/envs/gradio/lib/python3.9/site-packages/transformers/modeling_utils.py:977: FutureWarning: The `device` argument is deprecated and will be removed in v5 of Transformers. + warnings.warn( +/home/niki/anaconda3/envs/gradio/lib/python3.9/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/home/niki/anaconda3/envs/gradio/lib/python3.9/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None + warnings.warn( +[tensor([], device='cuda:0')] +0 +['[CLS]', 'cupboard', '##s', '.', '[SEP]'] +[0, 1, 2, 3, 4] +torch.Size([900, 256]) +torch.Size([900, 5]) +[] +/home/niki/anaconda3/envs/gradio/lib/python3.9/site-packages/transformers/modeling_utils.py:977: FutureWarning: The `device` argument is deprecated and will be removed in v5 of Transformers. + warnings.warn( +/home/niki/anaconda3/envs/gradio/lib/python3.9/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/home/niki/anaconda3/envs/gradio/lib/python3.9/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None + warnings.warn( +[tensor([], device='cuda:0')] +0 +['[CLS]', 'laptop', '.', '[SEP]'] +[0, 1, 2, 3] +torch.Size([900, 256]) +torch.Size([900, 4]) +[] +/home/niki/anaconda3/envs/gradio/lib/python3.9/site-packages/transformers/modeling_utils.py:977: FutureWarning: The `device` argument is deprecated and will be removed in v5 of Transformers. + warnings.warn( +/home/niki/anaconda3/envs/gradio/lib/python3.9/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/home/niki/anaconda3/envs/gradio/lib/python3.9/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None + warnings.warn( +[tensor([], device='cuda:0')] +0 +['[CLS]', 'finger', '.', '[SEP]'] +[0, 1, 2, 3] +torch.Size([900, 256]) +torch.Size([900, 4]) +[] +/home/niki/anaconda3/envs/gradio/lib/python3.9/site-packages/transformers/modeling_utils.py:977: FutureWarning: The `device` argument is deprecated and will be removed in v5 of Transformers. + warnings.warn( +/home/niki/anaconda3/envs/gradio/lib/python3.9/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/home/niki/anaconda3/envs/gradio/lib/python3.9/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None + warnings.warn( +[tensor([], device='cuda:0')] +0 +['[CLS]', 'bird', '.', '[SEP]'] +[0, 1, 2, 3] +torch.Size([900, 256]) +torch.Size([900, 4]) +[] +/home/niki/anaconda3/envs/gradio/lib/python3.9/site-packages/transformers/modeling_utils.py:977: FutureWarning: The `device` argument is deprecated and will be removed in v5 of Transformers. + warnings.warn( +/home/niki/anaconda3/envs/gradio/lib/python3.9/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/home/niki/anaconda3/envs/gradio/lib/python3.9/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None + warnings.warn( +[tensor([], device='cuda:0')] +0 +['[CLS]', 'dog', '.', '[SEP]'] +[0, 1, 2, 3] +torch.Size([900, 256]) +torch.Size([900, 4]) +[[712.0, 192.0, 876.0, 368.0]] +/home/niki/anaconda3/envs/gradio/lib/python3.9/site-packages/transformers/modeling_utils.py:977: FutureWarning: The `device` argument is deprecated and will be removed in v5 of Transformers. + warnings.warn( +/home/niki/anaconda3/envs/gradio/lib/python3.9/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/home/niki/anaconda3/envs/gradio/lib/python3.9/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None + warnings.warn( +[tensor([[659.0945, 177.7778, 810.9083, 340.7407]], device='cuda:0')] +1 +tensor([[ 101, 1008, 1012, 102]], device='cuda:0') +['[CLS]', '.', '[SEP]'] +[0, 1, 2] +torch.Size([900, 256]) +torch.Size([900, 3]) +[[712.0, 192.0, 876.0, 368.0], [525.0, 438.0, 631.0, 517.0], [918.0, 377.0, 1028.0, 474.0]] +/home/niki/anaconda3/envs/gradio/lib/python3.9/site-packages/transformers/modeling_utils.py:977: FutureWarning: The `device` argument is deprecated and will be removed in v5 of Transformers. + warnings.warn( +/home/niki/anaconda3/envs/gradio/lib/python3.9/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/home/niki/anaconda3/envs/gradio/lib/python3.9/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None + warnings.warn( +[tensor([[659.0945, 177.7778, 810.9083, 340.7407], + [485.9896, 405.5555, 584.1132, 478.7037], + [849.7875, 349.0741, 951.6139, 438.8889]], device='cuda:0')] +3 +tensor([[ 101, 1008, 1008, 1008, 1012, 102]], device='cuda:0') +['[CLS]', '.', '[SEP]'] +[0, 1, 2] +torch.Size([900, 256]) +torch.Size([900, 3]) +[[712.0, 192.0, 876.0, 368.0], [525.0, 438.0, 631.0, 517.0], [918.0, 377.0, 1028.0, 474.0]] +/home/niki/anaconda3/envs/gradio/lib/python3.9/site-packages/transformers/modeling_utils.py:977: FutureWarning: The `device` argument is deprecated and will be removed in v5 of Transformers. + warnings.warn( +/home/niki/anaconda3/envs/gradio/lib/python3.9/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/home/niki/anaconda3/envs/gradio/lib/python3.9/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None + warnings.warn( +[tensor([[659.0945, 177.7778, 810.9083, 340.7407], + [485.9896, 405.5555, 584.1132, 478.7037], + [849.7875, 349.0741, 951.6139, 438.8889]], device='cuda:0')] +3 +tensor([[ 101, 3899, 1008, 1008, 1008, 1012, 102]], device='cuda:0') +['[CLS]', 'dog', '.', '[SEP]'] +[0, 1, 2, 3] +torch.Size([900, 256]) +torch.Size([900, 4]) +/scratch/shared/beegfs/nikian/anaconda/envs/countgd-app/lib/python3.9/site-packages/torch/functional.py:512: UserWarning: torch.meshgrid: in an upcoming release, it will be required to pass the indexing argument. (Triggered internally at ../aten/src/ATen/native/TensorShape.cpp:3587.) + return _VF.meshgrid(tensors, **kwargs) # type: ignore[attr-defined] +final text_encoder_type: checkpoints/bert-base-uncased +load tokenizer done. +Some weights of BertModel were not initialized from the model checkpoint at checkpoints/bert-base-uncased and are newly initialized: ['pooler.dense.bias', 'pooler.dense.weight'] +You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference. +final text_encoder_type: checkpoints/bert-base-uncased +load tokenizer done. +Running on local URL: http://127.0.0.1:7860 +huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks... +To disable this warning, you can either: + - Avoid using `tokenizers` before the fork if possible + - Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false) +Running on public URL: https://5899ead67713f124a4.gradio.live + +This share link expires in 72 hours. For free permanent hosting and GPU upgrades, run `gradio deploy` from Terminal to deploy to Spaces (https://huggingface.co/spaces) +state: [] +[] +/scratch/shared/beegfs/nikian/anaconda/envs/countgd-app/lib/python3.9/site-packages/transformers/modeling_utils.py:1052: FutureWarning: The `device` argument is deprecated and will be removed in v5 of Transformers. + warnings.warn( +/scratch/shared/beegfs/nikian/anaconda/envs/countgd-app/lib/python3.9/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/scratch/shared/beegfs/nikian/anaconda/envs/countgd-app/lib/python3.9/site-packages/torch/utils/checkpoint.py:91: UserWarning: None of the inputs have requires_grad=True. Gradients will be None + warnings.warn( +/scratch/shared/beegfs/nikian/anaconda/envs/countgd-app/lib/python3.9/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/scratch/shared/beegfs/nikian/anaconda/envs/countgd-app/lib/python3.9/site-packages/torch/utils/checkpoint.py:91: UserWarning: None of the inputs have requires_grad=True. Gradients will be None + warnings.warn( +[tensor([], device='cuda:0')] +0 +/scratch/shared/beegfs/nikian/anaconda/envs/countgd-app/lib/python3.9/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/scratch/shared/beegfs/nikian/anaconda/envs/countgd-app/lib/python3.9/site-packages/torch/utils/checkpoint.py:91: UserWarning: None of the inputs have requires_grad=True. Gradients will be None + warnings.warn( +/scratch/shared/beegfs/nikian/anaconda/envs/countgd-app/lib/python3.9/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/scratch/shared/beegfs/nikian/anaconda/envs/countgd-app/lib/python3.9/site-packages/torch/utils/checkpoint.py:91: UserWarning: None of the inputs have requires_grad=True. Gradients will be None + warnings.warn( +['[CLS]', 'strawberry', '.', '[SEP]'] +[0, 1, 2, 3] +torch.Size([900, 256]) +torch.Size([900, 4]) +state: [, ] +[[161.0, 69.0, 205.0, 127.0]] +/scratch/shared/beegfs/nikian/anaconda/envs/countgd-app/lib/python3.9/site-packages/transformers/modeling_utils.py:1052: FutureWarning: The `device` argument is deprecated and will be removed in v5 of Transformers. + warnings.warn( +/scratch/shared/beegfs/nikian/anaconda/envs/countgd-app/lib/python3.9/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/scratch/shared/beegfs/nikian/anaconda/envs/countgd-app/lib/python3.9/site-packages/torch/utils/checkpoint.py:91: UserWarning: None of the inputs have requires_grad=True. Gradients will be None + warnings.warn( +[tensor([[335.3581, 143.7500, 427.0087, 264.5833]], device='cuda:0')] +1 +tensor([[ 101, 16876, 1008, 1012, 102]], device='cuda:0') +['[CLS]', 'strawberry', '.', '[SEP]'] +[0, 1, 2, 3] +torch.Size([900, 256]) +torch.Size([900, 4]) +[] +/scratch/shared/beegfs/nikian/anaconda/envs/countgd-app/lib/python3.9/site-packages/transformers/modeling_utils.py:1052: FutureWarning: The `device` argument is deprecated and will be removed in v5 of Transformers. + warnings.warn( +/scratch/shared/beegfs/nikian/anaconda/envs/countgd-app/lib/python3.9/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/scratch/shared/beegfs/nikian/anaconda/envs/countgd-app/lib/python3.9/site-packages/torch/utils/checkpoint.py:91: UserWarning: None of the inputs have requires_grad=True. Gradients will be None + warnings.warn( +[tensor([], device='cuda:0')] +0 +['[CLS]', 'fish', '.', '[SEP]'] +[0, 1, 2, 3] +torch.Size([900, 256]) +torch.Size([900, 4]) +[[382.0, 14.0, 476.0, 60.0]] +/scratch/shared/beegfs/nikian/anaconda/envs/countgd-app/lib/python3.9/site-packages/transformers/modeling_utils.py:1052: FutureWarning: The `device` argument is deprecated and will be removed in v5 of Transformers. + warnings.warn( +/scratch/shared/beegfs/nikian/anaconda/envs/countgd-app/lib/python3.9/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/scratch/shared/beegfs/nikian/anaconda/envs/countgd-app/lib/python3.9/site-packages/torch/utils/checkpoint.py:91: UserWarning: None of the inputs have requires_grad=True. Gradients will be None + warnings.warn( +[tensor([[570.8486, 20.9346, 711.3192, 89.7196]], device='cuda:0')] +1 +tensor([[ 101, 3869, 1008, 1012, 102]], device='cuda:0') +['[CLS]', 'fish', '.', '[SEP]'] +[0, 1, 2, 3] +torch.Size([900, 256]) +torch.Size([900, 4]) +[[382.0, 14.0, 476.0, 60.0], [320.0, 410.0, 447.0, 475.0]] +/scratch/shared/beegfs/nikian/anaconda/envs/countgd-app/lib/python3.9/site-packages/transformers/modeling_utils.py:1052: FutureWarning: The `device` argument is deprecated and will be removed in v5 of Transformers. + warnings.warn( +/scratch/shared/beegfs/nikian/anaconda/envs/countgd-app/lib/python3.9/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/scratch/shared/beegfs/nikian/anaconda/envs/countgd-app/lib/python3.9/site-packages/torch/utils/checkpoint.py:91: UserWarning: None of the inputs have requires_grad=True. Gradients will be None + warnings.warn( +[tensor([[570.8486, 20.9346, 711.3192, 89.7196], + [478.1978, 613.0841, 667.9825, 710.2804]], device='cuda:0')] +2 +tensor([[ 101, 3869, 1008, 1008, 1012, 102]], device='cuda:0') +['[CLS]', 'fish', '.', '[SEP]'] +[0, 1, 2, 3] +torch.Size([900, 256]) +torch.Size([900, 4]) +[] +/scratch/shared/beegfs/nikian/anaconda/envs/countgd-app/lib/python3.9/site-packages/transformers/modeling_utils.py:1052: FutureWarning: The `device` argument is deprecated and will be removed in v5 of Transformers. + warnings.warn( +/scratch/shared/beegfs/nikian/anaconda/envs/countgd-app/lib/python3.9/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/scratch/shared/beegfs/nikian/anaconda/envs/countgd-app/lib/python3.9/site-packages/torch/utils/checkpoint.py:91: UserWarning: None of the inputs have requires_grad=True. Gradients will be None + warnings.warn( +[tensor([], device='cuda:0')] +0 +['[CLS]', 'deer', '.', '[SEP]'] +[0, 1, 2, 3] +torch.Size([900, 256]) +torch.Size([900, 4]) +[[181.0, 51.0, 266.0, 131.0]] +/scratch/shared/beegfs/nikian/anaconda/envs/countgd-app/lib/python3.9/site-packages/transformers/modeling_utils.py:1052: FutureWarning: The `device` argument is deprecated and will be removed in v5 of Transformers. + warnings.warn( +/scratch/shared/beegfs/nikian/anaconda/envs/countgd-app/lib/python3.9/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/scratch/shared/beegfs/nikian/anaconda/envs/countgd-app/lib/python3.9/site-packages/torch/utils/checkpoint.py:91: UserWarning: None of the inputs have requires_grad=True. Gradients will be None + warnings.warn( +[tensor([[376.7822, 106.2500, 553.7242, 272.9167]], device='cuda:0')] +1 +tensor([[ 101, 8448, 1008, 1012, 102]], device='cuda:0') +['[CLS]', 'deer', '.', '[SEP]'] +[0, 1, 2, 3] +torch.Size([900, 256]) +torch.Size([900, 4]) +[[181.0, 51.0, 266.0, 131.0]] +/scratch/shared/beegfs/nikian/anaconda/envs/countgd-app/lib/python3.9/site-packages/transformers/modeling_utils.py:1052: FutureWarning: The `device` argument is deprecated and will be removed in v5 of Transformers. + warnings.warn( +/scratch/shared/beegfs/nikian/anaconda/envs/countgd-app/lib/python3.9/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/scratch/shared/beegfs/nikian/anaconda/envs/countgd-app/lib/python3.9/site-packages/torch/utils/checkpoint.py:91: UserWarning: None of the inputs have requires_grad=True. Gradients will be None + warnings.warn( +/scratch/shared/beegfs/nikian/anaconda/envs/countgd-app/lib/python3.9/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/scratch/shared/beegfs/nikian/anaconda/envs/countgd-app/lib/python3.9/site-packages/torch/utils/checkpoint.py:91: UserWarning: None of the inputs have requires_grad=True. Gradients will be None + warnings.warn( +[tensor([[376.7822, 106.2500, 553.7242, 272.9167]], device='cuda:0')] +1 +tensor([[ 101, 8448, 1008, 1012, 102]], device='cuda:0') +['[CLS]', 'deer', '.', '[SEP]'] +[0, 1, 2, 3] +torch.Size([900, 256]) +torch.Size([900, 4]) +state: [] +[] +/scratch/shared/beegfs/nikian/anaconda/envs/countgd-app/lib/python3.9/site-packages/transformers/modeling_utils.py:1052: FutureWarning: The `device` argument is deprecated and will be removed in v5 of Transformers. + warnings.warn( +/scratch/shared/beegfs/nikian/anaconda/envs/countgd-app/lib/python3.9/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/scratch/shared/beegfs/nikian/anaconda/envs/countgd-app/lib/python3.9/site-packages/torch/utils/checkpoint.py:91: UserWarning: None of the inputs have requires_grad=True. Gradients will be None + warnings.warn( +[tensor([], device='cuda:0')] +0 +['[CLS]', 'strawberry', '.', '[SEP]'] +[0, 1, 2, 3] +torch.Size([900, 256]) +torch.Size([900, 4]) +state: [, ] +[[153.0, 75.0, 212.0, 141.0]] +/scratch/shared/beegfs/nikian/anaconda/envs/countgd-app/lib/python3.9/site-packages/transformers/modeling_utils.py:1052: FutureWarning: The `device` argument is deprecated and will be removed in v5 of Transformers. + warnings.warn( +/scratch/shared/beegfs/nikian/anaconda/envs/countgd-app/lib/python3.9/site-packages/torch/utils/checkpoint.py:464: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants. + warnings.warn( +/scratch/shared/beegfs/nikian/anaconda/envs/countgd-app/lib/python3.9/site-packages/torch/utils/checkpoint.py:91: UserWarning: None of the inputs have requires_grad=True. Gradients will be None + warnings.warn( +[tensor([[318.6943, 156.2500, 441.5895, 293.7500]], device='cuda:0')] +1 +tensor([[ 101, 16876, 1008, 1012, 102]], device='cuda:0') +['[CLS]', 'strawberry', '.', '[SEP]'] +[0, 1, 2, 3] +torch.Size([900, 256]) +torch.Size([900, 4]) diff --git a/paste-icon.jpg b/paste-icon.jpg new file mode 100644 index 0000000000000000000000000000000000000000..9068ed28cd98cf9d761d7b3bf5a5211e4e08d4c9 Binary files /dev/null and b/paste-icon.jpg differ diff --git a/prompt_demo.py b/prompt_demo.py new file mode 100644 index 0000000000000000000000000000000000000000..f350a6ecda723fe640633083be6db85c390cb4ac --- /dev/null +++ b/prompt_demo.py @@ -0,0 +1,9 @@ +import gradio as gr +from gradio_image_prompter import ImagePrompter + +demo = gr.Interface( + lambda prompts: (prompts["image"], prompts["points"]), + ImagePrompter(show_label=False), + [gr.Image(show_label=False), gr.Dataframe(label="Points")], +) +demo.launch() diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..2d71f9904a19017d3fb33f8fdfb1c855329b9f72 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,16 @@ +cython +submitit +scipy +termcolor +addict +yapf==0.40.1 +timm +torch +torchvision +transformers +numpy +opencv-python +supervision==0.6.0 +pycocotools +pyyaml>3.10 +colorlog diff --git a/stamp.jpg b/stamp.jpg new file mode 100644 index 0000000000000000000000000000000000000000..79b86d25acbd48d326c7a5dd42473f729cc7d481 Binary files /dev/null and b/stamp.jpg differ diff --git a/strawberry.jpg b/strawberry.jpg new file mode 100644 index 0000000000000000000000000000000000000000..17bbe4a39002d0305c43bc4135e907d9db1312e8 Binary files /dev/null and b/strawberry.jpg differ diff --git a/teaser-gradio.jpg b/teaser-gradio.jpg new file mode 100644 index 0000000000000000000000000000000000000000..3512178d69d19f453b2f1cb2ec0f57a23fb87baa Binary files /dev/null and b/teaser-gradio.jpg differ diff --git a/tmp/1a715c1ee3d10474ff9eff9861c9022908021fb3/image.webp b/tmp/1a715c1ee3d10474ff9eff9861c9022908021fb3/image.webp new file mode 100644 index 0000000000000000000000000000000000000000..ec3ab16af9b220b31113c01b522bdc86581798c1 Binary files /dev/null and b/tmp/1a715c1ee3d10474ff9eff9861c9022908021fb3/image.webp differ diff --git a/tmp/36184494d8697ee195c2a46de0993ceb74fc59c5/lego.jpg b/tmp/36184494d8697ee195c2a46de0993ceb74fc59c5/lego.jpg new file mode 100644 index 0000000000000000000000000000000000000000..8426603c979e7c54afff3399477d566a9642c58f Binary files /dev/null and b/tmp/36184494d8697ee195c2a46de0993ceb74fc59c5/lego.jpg differ diff --git a/tmp/36f1c9b302c79af52b553c02cee7a7be332095c7/image.webp b/tmp/36f1c9b302c79af52b553c02cee7a7be332095c7/image.webp new file mode 100644 index 0000000000000000000000000000000000000000..1f17929a01186b7b7f0e2168c7b758ac654a6e3c Binary files /dev/null and b/tmp/36f1c9b302c79af52b553c02cee7a7be332095c7/image.webp differ diff --git a/tmp/3b8b85346ce04f6052600a332a7c8772ec6490ec/image.webp b/tmp/3b8b85346ce04f6052600a332a7c8772ec6490ec/image.webp new file mode 100644 index 0000000000000000000000000000000000000000..8b2efdaec74cec64851cadf883ca37a1a52664db Binary files /dev/null and b/tmp/3b8b85346ce04f6052600a332a7c8772ec6490ec/image.webp differ diff --git a/tmp/3bb35afa1ecb21c4ab9ae1383d6205db0a8eb651/fish.jpg b/tmp/3bb35afa1ecb21c4ab9ae1383d6205db0a8eb651/fish.jpg new file mode 100644 index 0000000000000000000000000000000000000000..0196080967488e21e1e15934aaea45d75c1272f5 Binary files /dev/null and b/tmp/3bb35afa1ecb21c4ab9ae1383d6205db0a8eb651/fish.jpg differ diff --git a/tmp/3dd4739579603106bd709d4c42713a0d8c047a94/image.webp b/tmp/3dd4739579603106bd709d4c42713a0d8c047a94/image.webp new file mode 100644 index 0000000000000000000000000000000000000000..17b7b6e4ae335dcb9a30c946f42fefd7a28db136 Binary files /dev/null and b/tmp/3dd4739579603106bd709d4c42713a0d8c047a94/image.webp differ diff --git a/tmp/3e1eed4061df45c7018f612dd86d432b391af4a1/deer.jpg b/tmp/3e1eed4061df45c7018f612dd86d432b391af4a1/deer.jpg new file mode 100644 index 0000000000000000000000000000000000000000..a52bf1146c1b995cfcde3ed393487198c3771df7 Binary files /dev/null and b/tmp/3e1eed4061df45c7018f612dd86d432b391af4a1/deer.jpg differ diff --git a/tmp/42748bdd7020c9b840c768b6a49b2ad47f134800/bird-2.JPG b/tmp/42748bdd7020c9b840c768b6a49b2ad47f134800/bird-2.JPG new file mode 100644 index 0000000000000000000000000000000000000000..af61947e18fc81a225f906a1479febdc074ccfc2 Binary files /dev/null and b/tmp/42748bdd7020c9b840c768b6a49b2ad47f134800/bird-2.JPG differ diff --git a/tmp/503b3f551ae0713374e7ab540868991bbe2cd5db/image.webp b/tmp/503b3f551ae0713374e7ab540868991bbe2cd5db/image.webp new file mode 100644 index 0000000000000000000000000000000000000000..e0e67023cf058dc2db773af1fbf6171ed9a0079c Binary files /dev/null and b/tmp/503b3f551ae0713374e7ab540868991bbe2cd5db/image.webp differ diff --git a/tmp/66e32e79a1fd790c21f4a99cd06936990a13314c/women.jpg b/tmp/66e32e79a1fd790c21f4a99cd06936990a13314c/women.jpg new file mode 100644 index 0000000000000000000000000000000000000000..51b780e07a3bb582e2072f9905cea5c196c1a6de Binary files /dev/null and b/tmp/66e32e79a1fd790c21f4a99cd06936990a13314c/women.jpg differ diff --git a/tmp/6c2833ac0ebc263298899e5051ad29db929b8d6c/bird-1.JPG b/tmp/6c2833ac0ebc263298899e5051ad29db929b8d6c/bird-1.JPG new file mode 100644 index 0000000000000000000000000000000000000000..6975666d7dd574cdcd519601b4460f015353e632 Binary files /dev/null and b/tmp/6c2833ac0ebc263298899e5051ad29db929b8d6c/bird-1.JPG differ diff --git a/tmp/6c60bd2f508eaf9bd413810af2d329ba6764ac4d/image.webp b/tmp/6c60bd2f508eaf9bd413810af2d329ba6764ac4d/image.webp new file mode 100644 index 0000000000000000000000000000000000000000..102ea92f12421ac6bac6752d7db73ca830891d5a Binary files /dev/null and b/tmp/6c60bd2f508eaf9bd413810af2d329ba6764ac4d/image.webp differ diff --git a/tmp/6e5d3e5f2218a4149a83d283977adea695be68b4/image.webp b/tmp/6e5d3e5f2218a4149a83d283977adea695be68b4/image.webp new file mode 100644 index 0000000000000000000000000000000000000000..81bdae9e86492e1b4a7906134afdfd3a8175f16b Binary files /dev/null and b/tmp/6e5d3e5f2218a4149a83d283977adea695be68b4/image.webp differ diff --git a/tmp/76f85609d166b69a7d8106ec3162ebda85dfa92b/egg.jpg b/tmp/76f85609d166b69a7d8106ec3162ebda85dfa92b/egg.jpg new file mode 100644 index 0000000000000000000000000000000000000000..bb8a3e81c5ee2d2fb845785e68c5cb689a655ba3 Binary files /dev/null and b/tmp/76f85609d166b69a7d8106ec3162ebda85dfa92b/egg.jpg differ diff --git a/tmp/86276da8c57c5334496b66985fe3eb2876d739b9/image.webp b/tmp/86276da8c57c5334496b66985fe3eb2876d739b9/image.webp new file mode 100644 index 0000000000000000000000000000000000000000..db7a7f062d5bf0bc9dcde756802f6877c54fe489 Binary files /dev/null and b/tmp/86276da8c57c5334496b66985fe3eb2876d739b9/image.webp differ diff --git a/tmp/8a317a631fd8776cc48a944ede92fa94c8cffe46/apple.jpg b/tmp/8a317a631fd8776cc48a944ede92fa94c8cffe46/apple.jpg new file mode 100644 index 0000000000000000000000000000000000000000..993cc2d709946bc68cd13c87854e2d3c0908ad2f Binary files /dev/null and b/tmp/8a317a631fd8776cc48a944ede92fa94c8cffe46/apple.jpg differ diff --git a/tmp/8af38646ed5846ee3219a7d37627bc6481d41aae/image.webp b/tmp/8af38646ed5846ee3219a7d37627bc6481d41aae/image.webp new file mode 100644 index 0000000000000000000000000000000000000000..5b2c46e9dc3f9f0b4ff7749b3ab33ec987d0a432 Binary files /dev/null and b/tmp/8af38646ed5846ee3219a7d37627bc6481d41aae/image.webp differ diff --git a/tmp/8de903563820d20e659c213cfba76c6e4d7d1954/image.webp b/tmp/8de903563820d20e659c213cfba76c6e4d7d1954/image.webp new file mode 100644 index 0000000000000000000000000000000000000000..9041bcb05aacc02b49926ce19db97cc67f060f3b Binary files /dev/null and b/tmp/8de903563820d20e659c213cfba76c6e4d7d1954/image.webp differ diff --git a/tmp/a05bd162195ea0562a8f24c68c46902d64152e04/image.webp b/tmp/a05bd162195ea0562a8f24c68c46902d64152e04/image.webp new file mode 100644 index 0000000000000000000000000000000000000000..890f8ec4d65d1a2952061a53f6511f483031a5ea Binary files /dev/null and b/tmp/a05bd162195ea0562a8f24c68c46902d64152e04/image.webp differ diff --git a/tmp/a3fe29df57d42f2cf37bcfdc2e86ec306cae13d2/stamp.jpg b/tmp/a3fe29df57d42f2cf37bcfdc2e86ec306cae13d2/stamp.jpg new file mode 100644 index 0000000000000000000000000000000000000000..79b86d25acbd48d326c7a5dd42473f729cc7d481 Binary files /dev/null and b/tmp/a3fe29df57d42f2cf37bcfdc2e86ec306cae13d2/stamp.jpg differ diff --git a/tmp/be0306249ad901142921e4a8028429019f0295ed/image.webp b/tmp/be0306249ad901142921e4a8028429019f0295ed/image.webp new file mode 100644 index 0000000000000000000000000000000000000000..9490fe3bf20dccebfc64d7a387742d59d48c88cc Binary files /dev/null and b/tmp/be0306249ad901142921e4a8028429019f0295ed/image.webp differ diff --git a/tmp/c49c153786614fc7f20a129a8f7084435fec47e4/image.webp b/tmp/c49c153786614fc7f20a129a8f7084435fec47e4/image.webp new file mode 100644 index 0000000000000000000000000000000000000000..31d9f1a3ec5f3da0f4f8f46ec5f2ba8cdb15ae19 Binary files /dev/null and b/tmp/c49c153786614fc7f20a129a8f7084435fec47e4/image.webp differ diff --git a/tmp/cf2ba4b285b58b64573cdd0289d4829c7bc78e89/image.webp b/tmp/cf2ba4b285b58b64573cdd0289d4829c7bc78e89/image.webp new file mode 100644 index 0000000000000000000000000000000000000000..bcb2e00554c795c47525ab2041b2346fd134a68b Binary files /dev/null and b/tmp/cf2ba4b285b58b64573cdd0289d4829c7bc78e89/image.webp differ diff --git a/tmp/d4f99749f19627c40d6a01687f9c732f33672258/image.webp b/tmp/d4f99749f19627c40d6a01687f9c732f33672258/image.webp new file mode 100644 index 0000000000000000000000000000000000000000..9d8c0741b09800e346ec43b31c68d7ff5032d4f5 Binary files /dev/null and b/tmp/d4f99749f19627c40d6a01687f9c732f33672258/image.webp differ diff --git a/tmp/df54e3df24582baab8254abe24542a49c3b0a0f7/image.webp b/tmp/df54e3df24582baab8254abe24542a49c3b0a0f7/image.webp new file mode 100644 index 0000000000000000000000000000000000000000..c34710dfe73476f053caa54c3367bcc40cb4ebfd Binary files /dev/null and b/tmp/df54e3df24582baab8254abe24542a49c3b0a0f7/image.webp differ diff --git a/tmp/e081d70cc56c7bb9829982f92f6947d0ae994adc/green-pea.jpg b/tmp/e081d70cc56c7bb9829982f92f6947d0ae994adc/green-pea.jpg new file mode 100644 index 0000000000000000000000000000000000000000..fa36364f74b98e909a3d634d2d39e4c98885f206 Binary files /dev/null and b/tmp/e081d70cc56c7bb9829982f92f6947d0ae994adc/green-pea.jpg differ diff --git a/tmp/e3bcc0df9f10df729fc0d74bc0e05cb4edf1f47b/balloon.jpg b/tmp/e3bcc0df9f10df729fc0d74bc0e05cb4edf1f47b/balloon.jpg new file mode 100644 index 0000000000000000000000000000000000000000..b492dd11a01260ec9693e6e251100a6eec657bd9 Binary files /dev/null and b/tmp/e3bcc0df9f10df729fc0d74bc0e05cb4edf1f47b/balloon.jpg differ diff --git a/tmp/f1900149b5b168ead2bc093e772bf7b63c4884ee/strawberry.jpg b/tmp/f1900149b5b168ead2bc093e772bf7b63c4884ee/strawberry.jpg new file mode 100644 index 0000000000000000000000000000000000000000..17bbe4a39002d0305c43bc4135e907d9db1312e8 Binary files /dev/null and b/tmp/f1900149b5b168ead2bc093e772bf7b63c4884ee/strawberry.jpg differ diff --git a/tools/GroundingDINO_SwinB_cfg.py b/tools/GroundingDINO_SwinB_cfg.py new file mode 100644 index 0000000000000000000000000000000000000000..f490c4bbd598a35de43d36ceafcbd769e7ff21bf --- /dev/null +++ b/tools/GroundingDINO_SwinB_cfg.py @@ -0,0 +1,43 @@ +batch_size = 1 +modelname = "groundingdino" +backbone = "swin_B_384_22k" +position_embedding = "sine" +pe_temperatureH = 20 +pe_temperatureW = 20 +return_interm_indices = [1, 2, 3] +backbone_freeze_keywords = None +enc_layers = 6 +dec_layers = 6 +pre_norm = False +dim_feedforward = 2048 +hidden_dim = 256 +dropout = 0.0 +nheads = 8 +num_queries = 900 +query_dim = 4 +num_patterns = 0 +num_feature_levels = 4 +enc_n_points = 4 +dec_n_points = 4 +two_stage_type = "standard" +two_stage_bbox_embed_share = False +two_stage_class_embed_share = False +transformer_activation = "relu" +dec_pred_bbox_embed_share = True +dn_box_noise_scale = 1.0 +dn_label_noise_ratio = 0.5 +dn_label_coef = 1.0 +dn_bbox_coef = 1.0 +embed_init_tgt = True +dn_labelbook_size = 2000 +max_text_len = 256 +text_encoder_type = "bert-base-uncased" +use_text_enhancer = True +use_fusion_layer = True +use_checkpoint = True +use_transformer_ckpt = True +use_text_cross_attention = True +text_dropout = 0.0 +fusion_dropout = 0.0 +fusion_droppath = 0.1 +sub_sentence_present = True diff --git a/tools/GroundingDINO_SwinT_OGC.py b/tools/GroundingDINO_SwinT_OGC.py new file mode 100644 index 0000000000000000000000000000000000000000..9158d5f6260ec74bded95377d382387430d7cd70 --- /dev/null +++ b/tools/GroundingDINO_SwinT_OGC.py @@ -0,0 +1,43 @@ +batch_size = 1 +modelname = "groundingdino" +backbone = "swin_T_224_1k" +position_embedding = "sine" +pe_temperatureH = 20 +pe_temperatureW = 20 +return_interm_indices = [1, 2, 3] +backbone_freeze_keywords = None +enc_layers = 6 +dec_layers = 6 +pre_norm = False +dim_feedforward = 2048 +hidden_dim = 256 +dropout = 0.0 +nheads = 8 +num_queries = 900 +query_dim = 4 +num_patterns = 0 +num_feature_levels = 4 +enc_n_points = 4 +dec_n_points = 4 +two_stage_type = "standard" +two_stage_bbox_embed_share = False +two_stage_class_embed_share = False +transformer_activation = "relu" +dec_pred_bbox_embed_share = True +dn_box_noise_scale = 1.0 +dn_label_noise_ratio = 0.5 +dn_label_coef = 1.0 +dn_bbox_coef = 1.0 +embed_init_tgt = True +dn_labelbook_size = 2000 +max_text_len = 256 +text_encoder_type = "bert-base-uncased" +use_text_enhancer = True +use_fusion_layer = True +use_checkpoint = True +use_transformer_ckpt = True +use_text_cross_attention = True +text_dropout = 0.0 +fusion_dropout = 0.0 +fusion_droppath = 0.1 +sub_sentence_present = True diff --git a/tools/benchmark.py b/tools/benchmark.py new file mode 100644 index 0000000000000000000000000000000000000000..1f3dd570e2c4f24eba0255587f9ed03eb8ba83dc --- /dev/null +++ b/tools/benchmark.py @@ -0,0 +1,671 @@ +# ------------------------------------------------------------------------ +# Copyright (c) 2023 IDEA. All Rights Reserved. +# ------------------------------------------------------------------------ +# ------------------------------------------------------------------------ +# Copyright (c) 2021 megvii-model. All Rights Reserved. +# ------------------------------------------------------------------------ + +# taken from https://gist.github.com/fmassa/c0fbb9fe7bf53b533b5cc241f5c8234c with a few modifications + +# taken from detectron2 / fvcore with a few modifications +# https://github.com/facebookresearch/detectron2/blob/master/detectron2/utils/analysis.py +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved + + +from collections import OrderedDict, Counter, defaultdict +import json +import os +from posixpath import join +import sys + + +sys.path.append(os.path.dirname(sys.path[0])) + +import numpy as np +from numpy import prod +from itertools import zip_longest +import tqdm +import logging +import typing +import torch +import torch.nn as nn +from functools import partial +import time + +from util.slconfig import SLConfig + +from typing import Any, Callable, List, Optional, Union +from numbers import Number + +Handle = Callable[[List[Any], List[Any]], Union[typing.Counter[str], Number]] + +from main import build_model_main, get_args_parser as get_main_args_parser +from datasets import build_dataset + + +def get_shape(val: object) -> typing.List[int]: + """ + Get the shapes from a jit value object. + Args: + val (torch._C.Value): jit value object. + Returns: + list(int): return a list of ints. + """ + if val.isCompleteTensor(): # pyre-ignore + r = val.type().sizes() # pyre-ignore + if not r: + r = [1] + return r + elif val.type().kind() in ("IntType", "FloatType"): + return [1] + elif val.type().kind() in ("StringType",): + return [0] + elif val.type().kind() in ("ListType",): + return [1] + elif val.type().kind() in ("BoolType", "NoneType"): + return [0] + else: + raise ValueError() + + +def addmm_flop_jit( + inputs: typing.List[object], outputs: typing.List[object] +) -> typing.Counter[str]: + """ + This method counts the flops for fully connected layers with torch script. + Args: + inputs (list(torch._C.Value)): The input shape in the form of a list of + jit object. + outputs (list(torch._C.Value)): The output shape in the form of a list + of jit object. + Returns: + Counter: A Counter dictionary that records the number of flops for each + operation. + """ + # Count flop for nn.Linear + # inputs is a list of length 3. + input_shapes = [get_shape(v) for v in inputs[1:3]] + # input_shapes[0]: [batch size, input feature dimension] + # input_shapes[1]: [batch size, output feature dimension] + assert len(input_shapes[0]) == 2 + assert len(input_shapes[1]) == 2 + batch_size, input_dim = input_shapes[0] + output_dim = input_shapes[1][1] + flop = batch_size * input_dim * output_dim + flop_counter = Counter({"addmm": flop}) + return flop_counter + + +def bmm_flop_jit(inputs, outputs): + # Count flop for nn.Linear + # inputs is a list of length 3. + input_shapes = [get_shape(v) for v in inputs] + # input_shapes[0]: [batch size, input feature dimension] + # input_shapes[1]: [batch size, output feature dimension] + assert len(input_shapes[0]) == 3 + assert len(input_shapes[1]) == 3 + T, batch_size, input_dim = input_shapes[0] + output_dim = input_shapes[1][2] + flop = T * batch_size * input_dim * output_dim + flop_counter = Counter({"bmm": flop}) + return flop_counter + + +def basic_binary_op_flop_jit(inputs, outputs, name): + input_shapes = [get_shape(v) for v in inputs] + # for broadcasting + input_shapes = [s[::-1] for s in input_shapes] + max_shape = np.array(list(zip_longest(*input_shapes, fillvalue=1))).max(1) + flop = prod(max_shape) + flop_counter = Counter({name: flop}) + return flop_counter + + +def rsqrt_flop_jit(inputs, outputs): + input_shapes = [get_shape(v) for v in inputs] + flop = prod(input_shapes[0]) * 2 + flop_counter = Counter({"rsqrt": flop}) + return flop_counter + + +def dropout_flop_jit(inputs, outputs): + input_shapes = [get_shape(v) for v in inputs[:1]] + flop = prod(input_shapes[0]) + flop_counter = Counter({"dropout": flop}) + return flop_counter + + +def softmax_flop_jit(inputs, outputs): + # from https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/profiler/internal/flops_registry.py + input_shapes = [get_shape(v) for v in inputs[:1]] + flop = prod(input_shapes[0]) * 5 + flop_counter = Counter({"softmax": flop}) + return flop_counter + + +def _reduction_op_flop_jit(inputs, outputs, reduce_flops=1, finalize_flops=0): + input_shapes = [get_shape(v) for v in inputs] + output_shapes = [get_shape(v) for v in outputs] + + in_elements = prod(input_shapes[0]) + out_elements = prod(output_shapes[0]) + + num_flops = in_elements * reduce_flops + out_elements * ( + finalize_flops - reduce_flops + ) + + return num_flops + + +def conv_flop_count( + x_shape: typing.List[int], + w_shape: typing.List[int], + out_shape: typing.List[int], +) -> typing.Counter[str]: + """ + This method counts the flops for convolution. Note only multiplication is + counted. Computation for addition and bias is ignored. + Args: + x_shape (list(int)): The input shape before convolution. + w_shape (list(int)): The filter shape. + out_shape (list(int)): The output shape after convolution. + Returns: + Counter: A Counter dictionary that records the number of flops for each + operation. + """ + batch_size, Cin_dim, Cout_dim = x_shape[0], w_shape[1], out_shape[1] + out_size = prod(out_shape[2:]) + kernel_size = prod(w_shape[2:]) + flop = batch_size * out_size * Cout_dim * Cin_dim * kernel_size + flop_counter = Counter({"conv": flop}) + return flop_counter + + +def conv_flop_jit( + inputs: typing.List[object], outputs: typing.List[object] +) -> typing.Counter[str]: + """ + This method counts the flops for convolution using torch script. + Args: + inputs (list(torch._C.Value)): The input shape in the form of a list of + jit object before convolution. + outputs (list(torch._C.Value)): The output shape in the form of a list + of jit object after convolution. + Returns: + Counter: A Counter dictionary that records the number of flops for each + operation. + """ + # Inputs of Convolution should be a list of length 12. They represent: + # 0) input tensor, 1) convolution filter, 2) bias, 3) stride, 4) padding, + # 5) dilation, 6) transposed, 7) out_pad, 8) groups, 9) benchmark_cudnn, + # 10) deterministic_cudnn and 11) user_enabled_cudnn. + # import ipdb; ipdb.set_trace() + # assert len(inputs) == 12 + x, w = inputs[:2] + x_shape, w_shape, out_shape = ( + get_shape(x), + get_shape(w), + get_shape(outputs[0]), + ) + return conv_flop_count(x_shape, w_shape, out_shape) + + +def einsum_flop_jit( + inputs: typing.List[object], outputs: typing.List[object] +) -> typing.Counter[str]: + """ + This method counts the flops for the einsum operation. We currently support + two einsum operations: "nct,ncp->ntp" and "ntg,ncg->nct". + Args: + inputs (list(torch._C.Value)): The input shape in the form of a list of + jit object before einsum. + outputs (list(torch._C.Value)): The output shape in the form of a list + of jit object after einsum. + Returns: + Counter: A Counter dictionary that records the number of flops for each + operation. + """ + # Inputs of einsum should be a list of length 2. + # Inputs[0] stores the equation used for einsum. + # Inputs[1] stores the list of input shapes. + assert len(inputs) == 2 + equation = inputs[0].toIValue() # pyre-ignore + # Get rid of white space in the equation string. + equation = equation.replace(" ", "") + # Re-map equation so that same equation with different alphabet + # representations will look the same. + letter_order = OrderedDict((k, 0) for k in equation if k.isalpha()).keys() + mapping = {ord(x): 97 + i for i, x in enumerate(letter_order)} + equation = equation.translate(mapping) + input_shapes_jit = inputs[1].node().inputs() # pyre-ignore + input_shapes = [get_shape(v) for v in input_shapes_jit] + + if equation == "abc,abd->acd": + n, c, t = input_shapes[0] + p = input_shapes[-1][-1] + flop = n * c * t * p + flop_counter = Counter({"einsum": flop}) + return flop_counter + + elif equation == "abc,adc->adb": + n, t, g = input_shapes[0] + c = input_shapes[-1][1] + flop = n * t * g * c + flop_counter = Counter({"einsum": flop}) + return flop_counter + + else: + raise NotImplementedError("Unsupported einsum operation.") + + +def matmul_flop_jit( + inputs: typing.List[object], outputs: typing.List[object] +) -> typing.Counter[str]: + """ + This method counts the flops for matmul. + Args: + inputs (list(torch._C.Value)): The input shape in the form of a list of + jit object before matmul. + outputs (list(torch._C.Value)): The output shape in the form of a list + of jit object after matmul. + Returns: + Counter: A Counter dictionary that records the number of flops for each + operation. + """ + + # Inputs contains the shapes of two matrices. + input_shapes = [get_shape(v) for v in inputs] + assert len(input_shapes) == 2 + assert input_shapes[0][-1] == input_shapes[1][-2] + + dim_len = len(input_shapes[1]) + assert dim_len >= 2 + batch = 1 + for i in range(dim_len - 2): + assert input_shapes[0][i] == input_shapes[1][i] + batch *= input_shapes[0][i] + + # (b,m,c) x (b,c,n), flop = bmnc + flop = batch * input_shapes[0][-2] * input_shapes[0][-1] * input_shapes[1][-1] + flop_counter = Counter({"matmul": flop}) + return flop_counter + + +def batchnorm_flop_jit( + inputs: typing.List[object], outputs: typing.List[object] +) -> typing.Counter[str]: + """ + This method counts the flops for batch norm. + Args: + inputs (list(torch._C.Value)): The input shape in the form of a list of + jit object before batch norm. + outputs (list(torch._C.Value)): The output shape in the form of a list + of jit object after batch norm. + Returns: + Counter: A Counter dictionary that records the number of flops for each + operation. + """ + # Inputs[0] contains the shape of the input. + input_shape = get_shape(inputs[0]) + assert 2 <= len(input_shape) <= 5 + flop = prod(input_shape) * 4 + flop_counter = Counter({"batchnorm": flop}) + return flop_counter + + +def linear_flop_jit(inputs: List[Any], outputs: List[Any]) -> Number: + """ + Count flops for the aten::linear operator. + """ + # Inputs is a list of length 3; unlike aten::addmm, it is the first + # two elements that are relevant. + input_shapes = [get_shape(v) for v in inputs[0:2]] + # input_shapes[0]: [dim0, dim1, ..., input_feature_dim] + # input_shapes[1]: [output_feature_dim, input_feature_dim] + assert input_shapes[0][-1] == input_shapes[1][-1] + flops = prod(input_shapes[0]) * input_shapes[1][0] + flop_counter = Counter({"linear": flops}) + return flop_counter + + +def norm_flop_counter(affine_arg_index: int) -> Handle: + """ + Args: + affine_arg_index: index of the affine argument in inputs + """ + + def norm_flop_jit(inputs: List[Any], outputs: List[Any]) -> Number: + """ + Count flops for norm layers. + """ + # Inputs[0] contains the shape of the input. + input_shape = get_shape(inputs[0]) + has_affine = get_shape(inputs[affine_arg_index]) is not None + assert 2 <= len(input_shape) <= 5, input_shape + # 5 is just a rough estimate + flop = prod(input_shape) * (5 if has_affine else 4) + flop_counter = Counter({"norm": flop}) + return flop_counter + + return norm_flop_jit + + +def elementwise_flop_counter(input_scale: float = 1, output_scale: float = 0) -> Handle: + """ + Count flops by + input_tensor.numel() * input_scale + output_tensor.numel() * output_scale + + Args: + input_scale: scale of the input tensor (first argument) + output_scale: scale of the output tensor (first element in outputs) + """ + + def elementwise_flop(inputs: List[Any], outputs: List[Any]) -> Number: + ret = 0 + if input_scale != 0: + shape = get_shape(inputs[0]) + ret += input_scale * prod(shape) + if output_scale != 0: + shape = get_shape(outputs[0]) + ret += output_scale * prod(shape) + flop_counter = Counter({"elementwise": ret}) + return flop_counter + + return elementwise_flop + + +# A dictionary that maps supported operations to their flop count jit handles. +_SUPPORTED_OPS: typing.Dict[str, typing.Callable] = { + "aten::addmm": addmm_flop_jit, + "aten::_convolution": conv_flop_jit, + "aten::einsum": einsum_flop_jit, + "aten::matmul": matmul_flop_jit, + "aten::batch_norm": batchnorm_flop_jit, + "aten::bmm": bmm_flop_jit, + "aten::add": partial(basic_binary_op_flop_jit, name="aten::add"), + "aten::add_": partial(basic_binary_op_flop_jit, name="aten::add_"), + "aten::mul": partial(basic_binary_op_flop_jit, name="aten::mul"), + "aten::sub": partial(basic_binary_op_flop_jit, name="aten::sub"), + "aten::div": partial(basic_binary_op_flop_jit, name="aten::div"), + "aten::floor_divide": partial(basic_binary_op_flop_jit, name="aten::floor_divide"), + "aten::relu": partial(basic_binary_op_flop_jit, name="aten::relu"), + "aten::relu_": partial(basic_binary_op_flop_jit, name="aten::relu_"), + "aten::sigmoid": partial(basic_binary_op_flop_jit, name="aten::sigmoid"), + "aten::log": partial(basic_binary_op_flop_jit, name="aten::log"), + "aten::sum": partial(basic_binary_op_flop_jit, name="aten::sum"), + "aten::sin": partial(basic_binary_op_flop_jit, name="aten::sin"), + "aten::cos": partial(basic_binary_op_flop_jit, name="aten::cos"), + "aten::pow": partial(basic_binary_op_flop_jit, name="aten::pow"), + "aten::cumsum": partial(basic_binary_op_flop_jit, name="aten::cumsum"), + "aten::rsqrt": rsqrt_flop_jit, + "aten::softmax": softmax_flop_jit, + "aten::dropout": dropout_flop_jit, + "aten::linear": linear_flop_jit, + "aten::group_norm": norm_flop_counter(2), + "aten::layer_norm": norm_flop_counter(2), + "aten::instance_norm": norm_flop_counter(1), + "aten::upsample_nearest2d": elementwise_flop_counter(0, 1), + "aten::upsample_bilinear2d": elementwise_flop_counter(0, 4), + "aten::adaptive_avg_pool2d": elementwise_flop_counter(1, 0), + "aten::max_pool2d": elementwise_flop_counter(1, 0), + "aten::mm": matmul_flop_jit, +} + + +# A list that contains ignored operations. +_IGNORED_OPS: typing.List[str] = [ + "aten::Int", + "aten::__and__", + "aten::arange", + "aten::cat", + "aten::clamp", + "aten::clamp_", + "aten::contiguous", + "aten::copy_", + "aten::detach", + "aten::empty", + "aten::eq", + "aten::expand", + "aten::flatten", + "aten::floor", + "aten::full", + "aten::gt", + "aten::index", + "aten::index_put_", + "aten::max", + "aten::nonzero", + "aten::permute", + "aten::remainder", + "aten::reshape", + "aten::select", + "aten::gather", + "aten::topk", + "aten::meshgrid", + "aten::masked_fill", + "aten::linspace", + "aten::size", + "aten::slice", + "aten::split_with_sizes", + "aten::squeeze", + "aten::t", + "aten::to", + "aten::transpose", + "aten::unsqueeze", + "aten::view", + "aten::zeros", + "aten::zeros_like", + "aten::ones_like", + "aten::new_zeros", + "aten::all", + "prim::Constant", + "prim::Int", + "prim::ListConstruct", + "prim::ListUnpack", + "prim::NumToTensor", + "prim::TupleConstruct", + "aten::stack", + "aten::chunk", + "aten::repeat", + "aten::grid_sampler", + "aten::constant_pad_nd", +] + +_HAS_ALREADY_SKIPPED = False + + +def flop_count( + model: nn.Module, + inputs: typing.Tuple[object, ...], + whitelist: typing.Union[typing.List[str], None] = None, + customized_ops: typing.Union[typing.Dict[str, typing.Callable], None] = None, +) -> typing.DefaultDict[str, float]: + """ + Given a model and an input to the model, compute the Gflops of the given + model. Note the input should have a batch size of 1. + Args: + model (nn.Module): The model to compute flop counts. + inputs (tuple): Inputs that are passed to `model` to count flops. + Inputs need to be in a tuple. + whitelist (list(str)): Whitelist of operations that will be counted. It + needs to be a subset of _SUPPORTED_OPS. By default, the function + computes flops for all supported operations. + customized_ops (dict(str,Callable)) : A dictionary contains customized + operations and their flop handles. If customized_ops contains an + operation in _SUPPORTED_OPS, then the default handle in + _SUPPORTED_OPS will be overwritten. + Returns: + defaultdict: A dictionary that records the number of gflops for each + operation. + """ + # Copy _SUPPORTED_OPS to flop_count_ops. + # If customized_ops is provided, update _SUPPORTED_OPS. + flop_count_ops = _SUPPORTED_OPS.copy() + if customized_ops: + flop_count_ops.update(customized_ops) + + # If whitelist is None, count flops for all suported operations. + if whitelist is None: + whitelist_set = set(flop_count_ops.keys()) + else: + whitelist_set = set(whitelist) + + # Torch script does not support parallell torch models. + if isinstance( + model, + (nn.parallel.distributed.DistributedDataParallel, nn.DataParallel), + ): + model = model.module # pyre-ignore + + assert set(whitelist_set).issubset( + flop_count_ops + ), "whitelist needs to be a subset of _SUPPORTED_OPS and customized_ops." + assert isinstance(inputs, tuple), "Inputs need to be in a tuple." + + # Compatibility with torch.jit. + if hasattr(torch.jit, "get_trace_graph"): + trace, _ = torch.jit.get_trace_graph(model, inputs) + trace_nodes = trace.graph().nodes() + else: + trace, _ = torch.jit._get_trace_graph(model, inputs) + trace_nodes = trace.nodes() + + skipped_ops = Counter() + total_flop_counter = Counter() + + for node in trace_nodes: + kind = node.kind() + if kind not in whitelist_set: + # If the operation is not in _IGNORED_OPS, count skipped operations. + if kind not in _IGNORED_OPS: + skipped_ops[kind] += 1 + continue + + handle_count = flop_count_ops.get(kind, None) + if handle_count is None: + continue + + inputs, outputs = list(node.inputs()), list(node.outputs()) + flops_counter = handle_count(inputs, outputs) + total_flop_counter += flops_counter + + global _HAS_ALREADY_SKIPPED + if len(skipped_ops) > 0 and not _HAS_ALREADY_SKIPPED: + _HAS_ALREADY_SKIPPED = True + for op, freq in skipped_ops.items(): + logging.warning("Skipped operation {} {} time(s)".format(op, freq)) + + # Convert flop count to gigaflops. + final_count = defaultdict(float) + for op in total_flop_counter: + final_count[op] = total_flop_counter[op] / 1e9 + + return final_count + + +def get_dataset(coco_path): + """ + Gets the COCO dataset used for computing the flops on + """ + + class DummyArgs: + pass + + args = DummyArgs() + args.dataset_file = "coco" + args.coco_path = coco_path + args.masks = False + dataset = build_dataset(image_set="val", args=args) + return dataset + + +def warmup(model, inputs, N=10): + for i in range(N): + out = model(inputs) + torch.cuda.synchronize() + + +def measure_time(model, inputs, N=10): + warmup(model, inputs) + s = time.time() + for i in range(N): + out = model(inputs) + torch.cuda.synchronize() + t = (time.time() - s) / N + return t + + +def fmt_res(data): + # return data.mean(), data.std(), data.min(), data.max() + return { + "mean": data.mean(), + "std": data.std(), + "min": data.min(), + "max": data.max(), + } + + +def benchmark(): + _outputs = {} + main_args = get_main_args_parser().parse_args() + main_args.commad_txt = "Command: " + " ".join(sys.argv) + + # load cfg file and update the args + print("Loading config file from {}".format(main_args.config_file)) + cfg = SLConfig.fromfile(main_args.config_file) + if main_args.options is not None: + cfg.merge_from_dict(main_args.options) + cfg_dict = cfg._cfg_dict.to_dict() + args_vars = vars(main_args) + for k, v in cfg_dict.items(): + if k not in args_vars: + setattr(main_args, k, v) + else: + raise ValueError("Key {} can used by args only".format(k)) + + dataset = build_dataset("val", main_args) + model, _, _ = build_model_main(main_args) + n_parameters = sum(p.numel() for p in model.parameters() if p.requires_grad) + _outputs.update({"nparam": n_parameters}) + + model.cuda() + model.eval() + + warmup_step = 5 + total_step = 20 + + images = [] + for idx in range(total_step): + img, t = dataset[idx] + images.append(img) + + with torch.no_grad(): + tmp = [] + tmp2 = [] + for imgid, img in enumerate(tqdm.tqdm(images)): + inputs = [img.to("cuda")] + res = flop_count(model, (inputs,)) + t = measure_time(model, inputs) + tmp.append(sum(res.values())) + if imgid >= warmup_step: + tmp2.append(t) + _outputs.update({"detailed_flops": res}) + _outputs.update({"flops": fmt_res(np.array(tmp)), "time": fmt_res(np.array(tmp2))}) + + mean_infer_time = float(fmt_res(np.array(tmp2))["mean"]) + _outputs.update({"fps": 1 / mean_infer_time}) + + res = {"flops": fmt_res(np.array(tmp)), "time": fmt_res(np.array(tmp2))} + # print(res) + + output_file = os.path.join(main_args.output_dir, "flops", "log.txt") + os.makedirs(os.path.dirname(output_file), exist_ok=True) + with open(output_file, "a") as f: + f.write(main_args.commad_txt + "\n") + f.write(json.dumps(_outputs, indent=2) + "\n") + + return _outputs + + +if __name__ == "__main__": + res = benchmark() + print(json.dumps(res, indent=2)) diff --git a/tools/coco2odvg.py b/tools/coco2odvg.py new file mode 100644 index 0000000000000000000000000000000000000000..f2384001b7407866de2c93c85acc88587237dcda --- /dev/null +++ b/tools/coco2odvg.py @@ -0,0 +1,80 @@ +import argparse +import jsonlines +from tqdm import tqdm +import json +from pycocotools.coco import COCO + +# this id_map is only for coco dataset which has 80 classes used for training but 90 categories in total. +# which change the start label -> 0 +# {"0": "person", "1": "bicycle", "2": "car", "3": "motorcycle", "4": "airplane", "5": "bus", "6": "train", "7": "truck", "8": "boat", "9": "traffic light", "10": "fire hydrant", "11": "stop sign", "12": "parking meter", "13": "bench", "14": "bird", "15": "cat", "16": "dog", "17": "horse", "18": "sheep", "19": "cow", "20": "elephant", "21": "bear", "22": "zebra", "23": "giraffe", "24": "backpack", "25": "umbrella", "26": "handbag", "27": "tie", "28": "suitcase", "29": "frisbee", "30": "skis", "31": "snowboard", "32": "sports ball", "33": "kite", "34": "baseball bat", "35": "baseball glove", "36": "skateboard", "37": "surfboard", "38": "tennis racket", "39": "bottle", "40": "wine glass", "41": "cup", "42": "fork", "43": "knife", "44": "spoon", "45": "bowl", "46": "banana", "47": "apple", "48": "sandwich", "49": "orange", "50": "broccoli", "51": "carrot", "52": "hot dog", "53": "pizza", "54": "donut", "55": "cake", "56": "chair", "57": "couch", "58": "potted plant", "59": "bed", "60": "dining table", "61": "toilet", "62": "tv", "63": "laptop", "64": "mouse", "65": "remote", "66": "keyboard", "67": "cell phone", "68": "microwave", "69": "oven", "70": "toaster", "71": "sink", "72": "refrigerator", "73": "book", "74": "clock", "75": "vase", "76": "scissors", "77": "teddy bear", "78": "hair drier", "79": "toothbrush"} + +id_map = {0: 1, 1: 2, 2: 3, 3: 4, 4: 5, 5: 6, 6: 7, 7: 8, 8: 9, 9: 10, 10: 11, 11: 13, 12: 14, 13: 15, 14: 16, 15: 17, 16: 18, 17: 19, 18: 20, 19: 21, 20: 22, 21: 23, 22: 24, 23: 25, 24: 27, 25: 28, 26: 31, 27: 32, 28: 33, 29: 34, 30: 35, 31: 36, 32: 37, 33: 38, 34: 39, 35: 40, 36: 41, 37: 42, 38: 43, 39: 44, 40: 46, 41: 47, 42: 48, 43: 49, 44: 50, 45: 51, 46: 52, 47: 53, 48: 54, 49: 55, 50: 56, 51: 57, 52: 58, 53: 59, 54: 60, 55: 61, 56: 62, 57: 63, 58: 64, 59: 65, 60: 67, 61: 70, 62: 72, 63: 73, 64: 74, 65: 75, 66: 76, 67: 77, 68: 78, 69: 79, 70: 80, 71: 81, 72: 82, 73: 84, 74: 85, 75: 86, 76: 87, 77: 88, 78: 89, 79: 90} +key_list=list(id_map.keys()) +val_list=list(id_map.values()) + +def dump_label_map(output="./out.json"): + ori_map = {"1": "person", "2": "bicycle", "3": "car", "4": "motorcycle", "5": "airplane", "6": "bus", "7": "train", "8": "truck", "9": "boat", "10": "traffic light", "11": "fire hydrant", "13": "stop sign", "14": "parking meter", "15": "bench", "16": "bird", "17": "cat", "18": "dog", "19": "horse", "20": "sheep", "21": "cow", "22": "elephant", "23": "bear", "24": "zebra", "25": "giraffe", "27": "backpack", "28": "umbrella", "31": "handbag", "32": "tie", "33": "suitcase", "34": "frisbee", "35": "skis", "36": "snowboard", "37": "sports ball", "38": "kite", "39": "baseball bat", "40": "baseball glove", "41": "skateboard", "42": "surfboard", "43": "tennis racket", "44": "bottle", "46": "wine glass", "47": "cup", "48": "fork", "49": "knife", "50": "spoon", "51": "bowl", "52": "banana", "53": "apple", "54": "sandwich", "55": "orange", "56": "broccoli", "57": "carrot", "58": "hot dog", "59": "pizza", "60": "donut", "61": "cake", "62": "chair", "63": "couch", "64": "potted plant", "65": "bed", "67": "dining table", "70": "toilet", "72": "tv", "73": "laptop", "74": "mouse", "75": "remote", "76": "keyboard", "77": "cell phone", "78": "microwave", "79": "oven", "80": "toaster", "81": "sink", "82": "refrigerator", "84": "book", "85": "clock", "86": "vase", "87": "scissors", "88": "teddy bear", "89": "hair drier", "90": "toothbrush"} + new_map = {} + for key, value in ori_map.items(): + label = int(key) + ind=val_list.index(label) + label_trans = key_list[ind] + new_map[label_trans] = value + with open(output,"w") as f: + json.dump(new_map, f) + +def coco_to_xyxy(bbox): + x, y, width, height = bbox + x1 = round(x, 2) + y1 = round(y, 2) + x2 = round(x + width, 2) + y2 = round(y + height, 2) + return [x1, y1, x2, y2] + + +def coco2odvg(args): + coco = COCO(args.input) + cats = coco.loadCats(coco.getCatIds()) + nms = {cat['id']:cat['name'] for cat in cats} + metas = [] + + for img_id, img_info in tqdm(coco.imgs.items()): + ann_ids = coco.getAnnIds(imgIds=img_id) + instance_list = [] + for ann_id in ann_ids: + ann = coco.anns[ann_id] + bbox = ann['bbox'] + bbox_xyxy = coco_to_xyxy(bbox) + label = ann['category_id'] + category = nms[label] + ind=val_list.index(label) + label_trans = key_list[ind] + instance_list.append({ + "bbox": bbox_xyxy, + "label": label_trans, + "category": category + } + ) + metas.append( + { + "filename": img_info["file_name"], + "height": img_info["height"], + "width": img_info["width"], + "detection": { + "instances": instance_list + } + } + ) + print(" == dump meta ...") + with jsonlines.open(args.output, mode="w") as writer: + writer.write_all(metas) + print(" == done.") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser("coco to odvg format.", add_help=True) + parser.add_argument("--input", '-i', required=True, type=str, help="input list name") + parser.add_argument("--output", '-o', required=True, type=str, help="output list name") + args = parser.parse_args() + + coco2odvg(args) diff --git a/tools/flickr30ke2odvg.py b/tools/flickr30ke2odvg.py new file mode 100644 index 0000000000000000000000000000000000000000..b8ea91c77907b734280e751bd4f7bcddea9d7339 --- /dev/null +++ b/tools/flickr30ke2odvg.py @@ -0,0 +1,180 @@ +import xml.etree.ElementTree as ET +import jsonlines +import random +from tqdm import tqdm +import argparse +import os +import glob + +def get_sentence_data(fn): + """ + Parses a sentence file from the Flickr30K Entities dataset + + input: + fn - full file path to the sentence file to parse + + output: + a list of dictionaries for each sentence with the following fields: + sentence - the original sentence + phrases - a list of dictionaries for each phrase with the + following fields: + phrase - the text of the annotated phrase + first_word_index - the position of the first word of + the phrase in the sentence + phrase_id - an identifier for this phrase + phrase_type - a list of the coarse categories this + phrase belongs to + + """ + with open(fn, 'r') as f: + sentences = f.read().split('\n') + + annotations = [] + for sentence in sentences: + if not sentence: + continue + + first_word = [] + phrases = [] + phrase_id = [] + phrase_type = [] + words = [] + current_phrase = [] + add_to_phrase = False + for token in sentence.split(): + if add_to_phrase: + if token[-1] == ']': + add_to_phrase = False + token = token[:-1] + current_phrase.append(token) + phrases.append(' '.join(current_phrase)) + current_phrase = [] + else: + current_phrase.append(token) + + words.append(token) + else: + if token[0] == '[': + add_to_phrase = True + first_word.append(len(words)) + parts = token.split('/') + phrase_id.append(parts[1][3:]) + phrase_type.append(parts[2:]) + else: + words.append(token) + + sentence_data = {'sentence' : ' '.join(words), 'phrases' : []} + for index, phrase, p_id, p_type in zip(first_word, phrases, phrase_id, phrase_type): + sentence_data['phrases'].append({'first_word_index' : index, + 'phrase' : phrase, + 'phrase_id' : p_id, + 'phrase_type' : p_type}) + + annotations.append(sentence_data) + + return annotations + +def get_annotations(fn): + """ + Parses the xml files in the Flickr30K Entities dataset + + input: + fn - full file path to the annotations file to parse + + output: + dictionary with the following fields: + scene - list of identifiers which were annotated as + pertaining to the whole scene + nobox - list of identifiers which were annotated as + not being visible in the image + boxes - a dictionary where the fields are identifiers + and the values are its list of boxes in the + [xmin ymin xmax ymax] format + """ + tree = ET.parse(fn) + root = tree.getroot() + filename = root.findall('filename')[0].text + size_container = root.findall('size')[0] + anno_info = {'filename': filename, 'boxes' : {}, 'scene' : [], 'nobox' : []} + for size_element in size_container: + anno_info[size_element.tag] = int(size_element.text) + + for object_container in root.findall('object'): + for names in object_container.findall('name'): + box_id = names.text + box_container = object_container.findall('bndbox') + if len(box_container) > 0: + if box_id not in anno_info['boxes']: + anno_info['boxes'][box_id] = [] + xmin = int(box_container[0].findall('xmin')[0].text) - 1 + ymin = int(box_container[0].findall('ymin')[0].text) - 1 + xmax = int(box_container[0].findall('xmax')[0].text) - 1 + ymax = int(box_container[0].findall('ymax')[0].text) - 1 + anno_info['boxes'][box_id].append([xmin, ymin, xmax, ymax]) + else: + nobndbox = int(object_container.findall('nobndbox')[0].text) + if nobndbox > 0: + anno_info['nobox'].append(box_id) + + scene = int(object_container.findall('scene')[0].text) + if scene > 0: + anno_info['scene'].append(box_id) + + return anno_info + +def gen_record(sd, an): + filename = an["filename"] + caption = sd["sentence"] + regions = [] + for ph in sd["phrases"]: + if ph["phrase_id"] in an["boxes"]: + for box in an["boxes"][ph["phrase_id"]]: + regions.append( + { + "phrase": ph["phrase"], + "bbox": box + } + ) + if len(regions) < 1: + print("no phrase regions") + return None + return { + "filename": filename, + "height": an["height"], + "width": an["width"], + "grounding":{ + "caption": caption, + "regions": regions + } + } + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="flickr30k entities to ODVG List.") + parser.add_argument("--root", type=str, default="", help="Source anno root") + parser.add_argument("--output_file", type=str, default="flickr30k_entities_odvg.jsonl") + parser.add_argument("--osoi", action="store_true", default=False) + args = parser.parse_args() + print(args) + + odvg_anno = [] + sentence_list = os.path.join(args.root, "Sentences") + annotation_list = os.path.join(args.root, "Annotations") + sentence_list = sorted(glob.glob(sentence_list + "/*")) + annotation_list = sorted(glob.glob(annotation_list + "/*")) + len_anno = len(annotation_list) + for idx in tqdm(range(len_anno)): + sds = get_sentence_data(sentence_list[idx]) + an = get_annotations(annotation_list[idx]) + if args.osoi: + sd = sds[random.randint(0, len(sds)-1)] + x = gen_record(sd, an) + if x: + odvg_anno.append(x) + else: + for sd in sds: + x = gen_record(sd, an) + if x: + odvg_anno.append(x) + with jsonlines.open(args.output_file, mode="w") as fwriter: + fwriter.write_all(odvg_anno) \ No newline at end of file diff --git a/tools/grit2odvg.py b/tools/grit2odvg.py new file mode 100644 index 0000000000000000000000000000000000000000..761f26fdf4715848dd382a71f7ef029915edbee1 --- /dev/null +++ b/tools/grit2odvg.py @@ -0,0 +1,112 @@ +import jsonlines +from tqdm import tqdm +import random +import json +import os +from multiprocessing import Pool +from functools import partial +import emoji + +import argparse + +def clean_span(span): + span = span.rstrip() + span = span.replace('"', "'").replace('\"', "'").replace('“', "'").replace('”', "'") + span = span.replace('‘', "'").replace('’', "'").replace('–', "—") + if span.endswith('/') or span.endswith('.'): + span = span[:-1] + return span + +def check_caption(cap): + check_anno = cap["caption"].rstrip()[:-1] + if not str.isascii(check_anno): + return False + # "The view is better from here 🦅 (Chouf" wtf?? + check_list = {"↙️", "-", ",", " ", "*", "/", "$", "[CLS]", "[SEP]", "?"} + for ch in check_list: + if ch in check_anno: + return False + if '.' in check_anno[:-1]: + return False + if emoji.emoji_count(check_anno): + print(check_anno) + return False + return True + +def get_regions(nc, anno): + h = anno["height"] + w = anno["width"] + phrase = clean_span(anno["caption"][int(nc[0]):int(nc[1])]) + bbox = [round(nc[2]*w,2), round(nc[3]*h,2), round(nc[4]*w,2), round(nc[5]*h,2)] + return { + "bbox": bbox, + "phrase": phrase + } + + +def prepare_list(file_name: str, random_samples): + with open(file_name, "r") as f: + metas = [line.strip() for line in f] + num_of_files = len(metas) + print(num_of_files) + metas = random.sample(metas, random_samples) + num_of_files = len(metas) + print("after sample:", num_of_files) + return metas, num_of_files + + +def process_item(file, args): + with open(os.path.join(args.root, file)) as f: + anno = json.load(f) + if not check_caption(anno): + return None + noun_chunks = anno['noun_chunks'] + ref_exps = anno['ref_exps'] + regions = [] + random_num = random.random() + if random_num > 0.5: + for nc in noun_chunks: + region = get_regions(nc, anno) + if str.isascii(region["phrase"]): + regions.append(region) + else: + for re in ref_exps: + region = get_regions(re, anno) + if str.isascii(region["phrase"]): + regions.append(region) + if len(regions) < args.min_phrase: + return None + odvg_anno = { + "filename": f'{file.split(".")[0]}.jpg', + "height": anno["height"], + "width": anno["width"], + "grounding": { + "caption": clean_span(anno["caption"]), + "regions": regions + } + } + return odvg_anno + +if __name__ == "__main__": + # jsons = "/share_data/mllm/kosmos-2/GRIT-20M/anno/14m_anno.list" + # root = "/share_data/mllm/kosmos-2/GRIT-20M/data" + # output_name = "./girt_14m_odvg.jsonl" + parser = argparse.ArgumentParser(description="GRIT2ODVG List.") + parser.add_argument("--input_file", type=str, required=True) + parser.add_argument("--root", type=str, default="", help="Source image root") + parser.add_argument("--output_file", type=str, default="girt_14m_odvg.jsonl") + parser.add_argument("--random_samples", type=int, default=200000) + parser.add_argument("--chunk_or_ref", type=float, default=0.5) + parser.add_argument("--min_phrase", type=int, default=6) + parser.add_argument("--process_num", type=int, default=10, help="the number of processes") + args = parser.parse_args() + print(args) + metas, metas_len = prepare_list(args.input_file, args.random_samples) + odvg_anno = [] + func = partial(process_item, args=args) + with Pool(processes=args.process_num) as pool: + for result in tqdm(pool.imap(func=func, iterable=metas), total=len(metas)): + odvg_anno.append(result) + odvg_anno = list(filter(None, odvg_anno)) + with jsonlines.open(args.output_file, mode="w") as fwriter: + fwriter.write_all(odvg_anno) \ No newline at end of file diff --git a/tools/inference_on_a_image.py b/tools/inference_on_a_image.py new file mode 100644 index 0000000000000000000000000000000000000000..ca64bb714bb2d7844f02ccd777b8625bde7fa37d --- /dev/null +++ b/tools/inference_on_a_image.py @@ -0,0 +1,214 @@ +import argparse +import os +import numpy as np +import torch +from PIL import Image, ImageDraw, ImageFont + +# please make sure https://github.com/IDEA-Research/GroundingDINO is installed correctly. +import groundingdino.datasets.transforms as T +from groundingdino.models import build_model +from groundingdino.util import box_ops +from groundingdino.util.slconfig import SLConfig +from groundingdino.util.utils import clean_state_dict, get_phrases_from_posmap +from groundingdino.util.vl_utils import create_positive_map_from_span + + +def plot_boxes_to_image(image_pil, tgt): + H, W = tgt["size"] + boxes = tgt["boxes"] + labels = tgt["labels"] + assert len(boxes) == len(labels), "boxes and labels must have same length" + + draw = ImageDraw.Draw(image_pil) + mask = Image.new("L", image_pil.size, 0) + mask_draw = ImageDraw.Draw(mask) + + # draw boxes and masks + for box, label in zip(boxes, labels): + # from 0..1 to 0..W, 0..H + box = box * torch.Tensor([W, H, W, H]) + # from xywh to xyxy + box[:2] -= box[2:] / 2 + box[2:] += box[:2] + # random color + color = tuple(np.random.randint(0, 255, size=3).tolist()) + # draw + x0, y0, x1, y1 = box + x0, y0, x1, y1 = int(x0), int(y0), int(x1), int(y1) + + draw.rectangle([x0, y0, x1, y1], outline=color, width=6) + # draw.text((x0, y0), str(label), fill=color) + + font = ImageFont.load_default() + if hasattr(font, "getbbox"): + bbox = draw.textbbox((x0, y0), str(label), font) + else: + w, h = draw.textsize(str(label), font) + bbox = (x0, y0, w + x0, y0 + h) + # bbox = draw.textbbox((x0, y0), str(label)) + draw.rectangle(bbox, fill=color) + draw.text((x0, y0), str(label), fill="white") + + mask_draw.rectangle([x0, y0, x1, y1], fill=255, width=6) + + return image_pil, mask + + +def load_image(image_path): + # load image + image_pil = Image.open(image_path).convert("RGB") # load image + + transform = T.Compose( + [ + T.RandomResize([800], max_size=1333), + T.ToTensor(), + T.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]), + ] + ) + image, _ = transform(image_pil, None) # 3, h, w + return image_pil, image + + +def load_model(model_config_path, model_checkpoint_path, cpu_only=False): + args = SLConfig.fromfile(model_config_path) + args.device = "cuda" if not cpu_only else "cpu" + model = build_model(args) + checkpoint = torch.load(model_checkpoint_path, map_location="cpu") + load_res = model.load_state_dict(clean_state_dict(checkpoint["model"]), strict=False) + print(load_res) + _ = model.eval() + return model + + +def get_grounding_output(model, image, caption, box_threshold, text_threshold=None, with_logits=True, cpu_only=False, token_spans=None): + assert text_threshold is not None or token_spans is not None, "text_threshould and token_spans should not be None at the same time!" + caption = caption.lower() + caption = caption.strip() + if not caption.endswith("."): + caption = caption + "." + device = "cuda" if not cpu_only else "cpu" + model = model.to(device) + image = image.to(device) + with torch.no_grad(): + outputs = model(image[None], captions=[caption]) + logits = outputs["pred_logits"].sigmoid()[0] # (nq, 256) + boxes = outputs["pred_boxes"][0] # (nq, 4) + + # filter output + if token_spans is None: + logits_filt = logits.cpu().clone() + boxes_filt = boxes.cpu().clone() + filt_mask = logits_filt.max(dim=1)[0] > box_threshold + logits_filt = logits_filt[filt_mask] # num_filt, 256 + boxes_filt = boxes_filt[filt_mask] # num_filt, 4 + + # get phrase + tokenlizer = model.tokenizer + tokenized = tokenlizer(caption) + # build pred + pred_phrases = [] + for logit, box in zip(logits_filt, boxes_filt): + pred_phrase = get_phrases_from_posmap(logit > text_threshold, tokenized, tokenlizer) + if with_logits: + pred_phrases.append(pred_phrase + f"({str(logit.max().item())[:4]})") + else: + pred_phrases.append(pred_phrase) + else: + # given-phrase mode + positive_maps = create_positive_map_from_span( + model.tokenizer(text_prompt), + token_span=token_spans + ).to(image.device) # n_phrase, 256 + + logits_for_phrases = positive_maps @ logits.T # n_phrase, nq + all_logits = [] + all_phrases = [] + all_boxes = [] + for (token_span, logit_phr) in zip(token_spans, logits_for_phrases): + # get phrase + phrase = ' '.join([caption[_s:_e] for (_s, _e) in token_span]) + # get mask + filt_mask = logit_phr > box_threshold + # filt box + all_boxes.append(boxes[filt_mask]) + # filt logits + all_logits.append(logit_phr[filt_mask]) + if with_logits: + logit_phr_num = logit_phr[filt_mask] + all_phrases.extend([phrase + f"({str(logit.item())[:4]})" for logit in logit_phr_num]) + else: + all_phrases.extend([phrase for _ in range(len(filt_mask))]) + boxes_filt = torch.cat(all_boxes, dim=0).cpu() + pred_phrases = all_phrases + + + return boxes_filt, pred_phrases + + +if __name__ == "__main__": + + parser = argparse.ArgumentParser("Grounding DINO example", add_help=True) + parser.add_argument("--config_file", "-c", type=str, required=True, help="path to config file") + parser.add_argument( + "--checkpoint_path", "-p", type=str, required=True, help="path to checkpoint file" + ) + parser.add_argument("--image_path", "-i", type=str, required=True, help="path to image file") + parser.add_argument("--text_prompt", "-t", type=str, required=True, help="text prompt") + parser.add_argument( + "--output_dir", "-o", type=str, default="outputs", required=True, help="output directory" + ) + + parser.add_argument("--box_threshold", type=float, default=0.3, help="box threshold") + parser.add_argument("--text_threshold", type=float, default=0.25, help="text threshold") + parser.add_argument("--token_spans", type=str, default=None, help= + "The positions of start and end positions of phrases of interest. \ + For example, a caption is 'a cat and a dog', \ + if you would like to detect 'cat', the token_spans should be '[[[2, 5]], ]', since 'a cat and a dog'[2:5] is 'cat'. \ + if you would like to detect 'a cat', the token_spans should be '[[[0, 1], [2, 5]], ]', since 'a cat and a dog'[0:1] is 'a', and 'a cat and a dog'[2:5] is 'cat'. \ + ") + + parser.add_argument("--cpu-only", action="store_true", help="running on cpu only!, default=False") + args = parser.parse_args() + + # cfg + config_file = args.config_file # change the path of the model config file + checkpoint_path = args.checkpoint_path # change the path of the model + image_path = args.image_path + text_prompt = args.text_prompt + output_dir = args.output_dir + box_threshold = args.box_threshold + text_threshold = args.text_threshold + token_spans = args.token_spans + + # make dir + os.makedirs(output_dir, exist_ok=True) + # load image + image_pil, image = load_image(image_path) + # load model + model = load_model(config_file, checkpoint_path, cpu_only=args.cpu_only) + + # visualize raw image + image_pil.save(os.path.join(output_dir, "raw_image.jpg")) + + # set the text_threshold to None if token_spans is set. + if token_spans is not None: + text_threshold = None + print("Using token_spans. Set the text_threshold to None.") + + + # run model + boxes_filt, pred_phrases = get_grounding_output( + model, image, text_prompt, box_threshold, text_threshold, cpu_only=args.cpu_only, token_spans=token_spans + ) + + # visualize pred + size = image_pil.size + pred_dict = { + "boxes": boxes_filt, + "size": [size[1], size[0]], # H,W + "labels": pred_phrases, + } + image_with_box = plot_boxes_to_image(image_pil, pred_dict)[0] + save_path = os.path.join(output_dir, "pred.jpg") + image_with_box.save(save_path) + print(f"\n======================\n{save_path} saved.\nThe program runs successfully!") diff --git a/tools/v3det2odvg.py b/tools/v3det2odvg.py new file mode 100644 index 0000000000000000000000000000000000000000..93dd95a9579f8cc73c62e7ec7b587afa5c1d172f --- /dev/null +++ b/tools/v3det2odvg.py @@ -0,0 +1,69 @@ +import argparse +import jsonlines +from tqdm import tqdm +import json +from pycocotools.coco import COCO + +def dump_label_map(args): + coco = COCO(args.input) + cats = coco.loadCats(coco.getCatIds()) + nms = {cat['id']-1:cat['name'] for cat in cats} + with open(args.output,"w") as f: + json.dump(nms, f) + +def coco_to_xyxy(bbox): + x, y, width, height = bbox + x1 = round(x, 2) + y1 = round(y, 2) + x2 = round(x + width, 2) + y2 = round(y + height, 2) + return [x1, y1, x2, y2] + + +def coco2odvg(args): + coco = COCO(args.input) + cats = coco.loadCats(coco.getCatIds()) + nms = {cat['id']:cat['name'] for cat in cats} + metas = [] + for img_id, img_info in tqdm(coco.imgs.items()): + ann_ids = coco.getAnnIds(imgIds=img_id) + instance_list = [] + for ann_id in ann_ids: + ann = coco.anns[ann_id] + bbox = ann['bbox'] + bbox_xyxy = coco_to_xyxy(bbox) + label = ann['category_id'] + category = nms[label] + instance_list.append({ + "bbox": bbox_xyxy, + "label": label - 1, # make sure start from 0 + "category": category + } + ) + metas.append( + { + "filename": img_info["file_name"], + "height": img_info["height"], + "width": img_info["width"], + "detection": { + "instances": instance_list + } + } + ) + print(" == dump meta ...") + with jsonlines.open(args.output, mode="w") as writer: + writer.write_all(metas) + print(" == done.") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser("coco to odvg format.", add_help=True) + parser.add_argument("--input", '-i', required=True, type=str, help="input list name") + parser.add_argument("--output", '-o', required=True, type=str, help="output list name") + parser.add_argument("--output_label_map", '-olm', action="store_true", help="output label map or not") + args = parser.parse_args() + + if args.output_label_map: + dump_label_map(args) + else: + coco2odvg(args) \ No newline at end of file diff --git a/upload-icon.jpg b/upload-icon.jpg new file mode 100644 index 0000000000000000000000000000000000000000..ac5aa509d1e6812a5ecb830d0ce1b8c49c0226af Binary files /dev/null and b/upload-icon.jpg differ diff --git a/util/__init__.py b/util/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..168f9979a4623806934b0ff1102ac166704e7dec --- /dev/null +++ b/util/__init__.py @@ -0,0 +1 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved diff --git a/util/__pycache__/__init__.cpython-38.pyc b/util/__pycache__/__init__.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..db444d10869380f6d4eefcd61703a76742dd8e4b Binary files /dev/null and b/util/__pycache__/__init__.cpython-38.pyc differ diff --git a/util/__pycache__/__init__.cpython-39.pyc b/util/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..fa250522b8310f1a501bca439ae63947fe302e11 Binary files /dev/null and b/util/__pycache__/__init__.cpython-39.pyc differ diff --git a/util/__pycache__/box_ops.cpython-38.pyc b/util/__pycache__/box_ops.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..743abfbc7577f3cd278763e24e5e87b85305135a Binary files /dev/null and b/util/__pycache__/box_ops.cpython-38.pyc differ diff --git a/util/__pycache__/box_ops.cpython-39.pyc b/util/__pycache__/box_ops.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9fc5def7a17cb940a7f9e6682366d3550eddca6e Binary files /dev/null and b/util/__pycache__/box_ops.cpython-39.pyc differ diff --git a/util/__pycache__/get_param_dicts.cpython-38.pyc b/util/__pycache__/get_param_dicts.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..870d6e564aea910bcbd690fee54155de3e0ea15b Binary files /dev/null and b/util/__pycache__/get_param_dicts.cpython-38.pyc differ diff --git a/util/__pycache__/get_param_dicts.cpython-39.pyc b/util/__pycache__/get_param_dicts.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..65aaf54cbec7956add344a4cc1a4864076ac8e5d Binary files /dev/null and b/util/__pycache__/get_param_dicts.cpython-39.pyc differ diff --git a/util/__pycache__/logger.cpython-38.pyc b/util/__pycache__/logger.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2819e3352601301044c371b78abf05b54823d405 Binary files /dev/null and b/util/__pycache__/logger.cpython-38.pyc differ diff --git a/util/__pycache__/logger.cpython-39.pyc b/util/__pycache__/logger.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7b74d4f6b127d0e980a52cd4e1b91f96367b0594 Binary files /dev/null and b/util/__pycache__/logger.cpython-39.pyc differ diff --git a/util/__pycache__/misc.cpython-38.pyc b/util/__pycache__/misc.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..bc65f11105f9ee4af89f7143c8e25ad4ce41238a Binary files /dev/null and b/util/__pycache__/misc.cpython-38.pyc differ diff --git a/util/__pycache__/misc.cpython-39.pyc b/util/__pycache__/misc.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ba7ade102cc1384048151be229134ac1a4be6e55 Binary files /dev/null and b/util/__pycache__/misc.cpython-39.pyc differ diff --git a/util/__pycache__/slconfig.cpython-38.pyc b/util/__pycache__/slconfig.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e2ed34af31b6ca5a3735f062264558829e37b8b2 Binary files /dev/null and b/util/__pycache__/slconfig.cpython-38.pyc differ diff --git a/util/__pycache__/slconfig.cpython-39.pyc b/util/__pycache__/slconfig.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..25cb5a04673a780118e0414b4f2b7b8b5cba3c49 Binary files /dev/null and b/util/__pycache__/slconfig.cpython-39.pyc differ diff --git a/util/__pycache__/utils.cpython-38.pyc b/util/__pycache__/utils.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9c7e56a65c40d3ce0f74bac17432769b56122b3c Binary files /dev/null and b/util/__pycache__/utils.cpython-38.pyc differ diff --git a/util/__pycache__/utils.cpython-39.pyc b/util/__pycache__/utils.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e5c931e098f76703aaa9658982211dd0c7b57296 Binary files /dev/null and b/util/__pycache__/utils.cpython-39.pyc differ diff --git a/util/__pycache__/visualizer.cpython-39.pyc b/util/__pycache__/visualizer.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..161e5c318d519b913b0e952d2d7362777795c193 Binary files /dev/null and b/util/__pycache__/visualizer.cpython-39.pyc differ diff --git a/util/box_loss.py b/util/box_loss.py new file mode 100644 index 0000000000000000000000000000000000000000..c0f94a5b5804a5e6a613785249d92f71594f4ab7 --- /dev/null +++ b/util/box_loss.py @@ -0,0 +1,112 @@ +# borrow from https://github.com/Zzh-tju/CIoU/blob/master/layers/modules/multibox_loss.py + +import torch, math + + + +def ciou(bboxes1, bboxes2): + bboxes1 = torch.sigmoid(bboxes1) + bboxes2 = torch.sigmoid(bboxes2) + rows = bboxes1.shape[0] + cols = bboxes2.shape[0] + cious = torch.zeros((rows, cols)) + if rows * cols == 0: + return cious + exchange = False + if bboxes1.shape[0] > bboxes2.shape[0]: + bboxes1, bboxes2 = bboxes2, bboxes1 + cious = torch.zeros((cols, rows)) + exchange = True + w1 = torch.exp(bboxes1[:, 2]) + h1 = torch.exp(bboxes1[:, 3]) + w2 = torch.exp(bboxes2[:, 2]) + h2 = torch.exp(bboxes2[:, 3]) + area1 = w1 * h1 + area2 = w2 * h2 + center_x1 = bboxes1[:, 0] + center_y1 = bboxes1[:, 1] + center_x2 = bboxes2[:, 0] + center_y2 = bboxes2[:, 1] + + inter_l = torch.max(center_x1 - w1 / 2,center_x2 - w2 / 2) + inter_r = torch.min(center_x1 + w1 / 2,center_x2 + w2 / 2) + inter_t = torch.max(center_y1 - h1 / 2,center_y2 - h2 / 2) + inter_b = torch.min(center_y1 + h1 / 2,center_y2 + h2 / 2) + inter_area = torch.clamp((inter_r - inter_l),min=0) * torch.clamp((inter_b - inter_t),min=0) + + c_l = torch.min(center_x1 - w1 / 2,center_x2 - w2 / 2) + c_r = torch.max(center_x1 + w1 / 2,center_x2 + w2 / 2) + c_t = torch.min(center_y1 - h1 / 2,center_y2 - h2 / 2) + c_b = torch.max(center_y1 + h1 / 2,center_y2 + h2 / 2) + + inter_diag = (center_x2 - center_x1)**2 + (center_y2 - center_y1)**2 + c_diag = torch.clamp((c_r - c_l),min=0)**2 + torch.clamp((c_b - c_t),min=0)**2 + + union = area1+area2-inter_area + u = (inter_diag) / c_diag + iou = inter_area / union + v = (4 / (math.pi ** 2)) * torch.pow((torch.atan(w2 / h2) - torch.atan(w1 / h1)), 2) + with torch.no_grad(): + S = (iou>0.5).float() + alpha= S*v/(1-iou+v) + cious = iou - u - alpha * v + cious = torch.clamp(cious,min=-1.0,max = 1.0) + if exchange: + cious = cious.T + return 1-cious + +def diou(bboxes1, bboxes2): + bboxes1 = torch.sigmoid(bboxes1) + bboxes2 = torch.sigmoid(bboxes2) + rows = bboxes1.shape[0] + cols = bboxes2.shape[0] + cious = torch.zeros((rows, cols)) + if rows * cols == 0: + return cious + exchange = False + if bboxes1.shape[0] > bboxes2.shape[0]: + bboxes1, bboxes2 = bboxes2, bboxes1 + cious = torch.zeros((cols, rows)) + exchange = True + w1 = torch.exp(bboxes1[:, 2]) + h1 = torch.exp(bboxes1[:, 3]) + w2 = torch.exp(bboxes2[:, 2]) + h2 = torch.exp(bboxes2[:, 3]) + area1 = w1 * h1 + area2 = w2 * h2 + center_x1 = bboxes1[:, 0] + center_y1 = bboxes1[:, 1] + center_x2 = bboxes2[:, 0] + center_y2 = bboxes2[:, 1] + + inter_l = torch.max(center_x1 - w1 / 2,center_x2 - w2 / 2) + inter_r = torch.min(center_x1 + w1 / 2,center_x2 + w2 / 2) + inter_t = torch.max(center_y1 - h1 / 2,center_y2 - h2 / 2) + inter_b = torch.min(center_y1 + h1 / 2,center_y2 + h2 / 2) + inter_area = torch.clamp((inter_r - inter_l),min=0) * torch.clamp((inter_b - inter_t),min=0) + + c_l = torch.min(center_x1 - w1 / 2,center_x2 - w2 / 2) + c_r = torch.max(center_x1 + w1 / 2,center_x2 + w2 / 2) + c_t = torch.min(center_y1 - h1 / 2,center_y2 - h2 / 2) + c_b = torch.max(center_y1 + h1 / 2,center_y2 + h2 / 2) + + inter_diag = (center_x2 - center_x1)**2 + (center_y2 - center_y1)**2 + c_diag = torch.clamp((c_r - c_l),min=0)**2 + torch.clamp((c_b - c_t),min=0)**2 + + union = area1+area2-inter_area + u = (inter_diag) / c_diag + iou = inter_area / union + dious = iou - u + dious = torch.clamp(dious,min=-1.0,max = 1.0) + if exchange: + dious = dious.T + return 1-dious + + +if __name__ == "__main__": + x = torch.rand(10, 4) + y = torch.rand(10,4) + import ipdb;ipdb.set_trace() + cxy = ciou(x, y) + dxy = diou(x, y) + print(cxy.shape, dxy.shape) diff --git a/util/box_ops.py b/util/box_ops.py new file mode 100644 index 0000000000000000000000000000000000000000..99ba6d0f67d536aa77bbf787c5ef19aeb56a50df --- /dev/null +++ b/util/box_ops.py @@ -0,0 +1,138 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved +""" +Utilities for bounding box manipulation and GIoU. +""" +import torch, os +from torchvision.ops.boxes import box_area + + +def box_cxcywh_to_xyxy(x): + x_c, y_c, w, h = x.unbind(-1) + b = [(x_c - 0.5 * w), (y_c - 0.5 * h), + (x_c + 0.5 * w), (y_c + 0.5 * h)] + return torch.stack(b, dim=-1) + + +def box_xyxy_to_cxcywh(x): + x0, y0, x1, y1 = x.unbind(-1) + b = [(x0 + x1) / 2, (y0 + y1) / 2, + (x1 - x0), (y1 - y0)] + return torch.stack(b, dim=-1) + + +# modified from torchvision to also return the union +def box_iou(boxes1, boxes2): + area1 = box_area(boxes1) + area2 = box_area(boxes2) + + + lt = torch.max(boxes1[:, None, :2], boxes2[:, :2]) # [N,M,2] + rb = torch.min(boxes1[:, None, 2:], boxes2[:, 2:]) # [N,M,2] + + wh = (rb - lt).clamp(min=0) # [N,M,2] + inter = wh[:, :, 0] * wh[:, :, 1] # [N,M] + + union = area1[:, None] + area2 - inter + + iou = inter / (union + 1e-6) + return iou, union + + +def generalized_box_iou(boxes1, boxes2): + """ + Generalized IoU from https://giou.stanford.edu/ + + The boxes should be in [x0, y0, x1, y1] format + + Returns a [N, M] pairwise matrix, where N = len(boxes1) + and M = len(boxes2) + """ + # degenerate boxes gives inf / nan results + # so do an early check + assert (boxes1[:, 2:] >= boxes1[:, :2]).all(), f"{boxes1}" + assert (boxes2[:, 2:] >= boxes2[:, :2]).all(), f"{boxes2}" + + iou, union = box_iou(boxes1, boxes2) + + lt = torch.min(boxes1[:, None, :2], boxes2[:, :2]) + rb = torch.max(boxes1[:, None, 2:], boxes2[:, 2:]) + + wh = (rb - lt).clamp(min=0) # [N,M,2] + area = wh[:, :, 0] * wh[:, :, 1] + + return iou - (area - union) / (area + 1e-6) + + + +# modified from torchvision to also return the union +def box_iou_pairwise(boxes1, boxes2): + area1 = box_area(boxes1) + area2 = box_area(boxes2) + + lt = torch.max(boxes1[:, :2], boxes2[:, :2]) # [N,2] + rb = torch.min(boxes1[:, 2:], boxes2[:, 2:]) # [N,2] + + wh = (rb - lt).clamp(min=0) # [N,2] + inter = wh[:, 0] * wh[:, 1] # [N] + + union = area1 + area2 - inter + + iou = inter / union + return iou, union + + +def generalized_box_iou_pairwise(boxes1, boxes2): + """ + Generalized IoU from https://giou.stanford.edu/ + + Input: + - boxes1, boxes2: N,4 + Output: + - giou: N, 4 + """ + # degenerate boxes gives inf / nan results + # so do an early check + assert (boxes1[:, 2:] >= boxes1[:, :2]).all() + assert (boxes2[:, 2:] >= boxes2[:, :2]).all() + assert boxes1.shape == boxes2.shape + iou, union = box_iou_pairwise(boxes1, boxes2) # N, 4 + + lt = torch.min(boxes1[:, :2], boxes2[:, :2]) + rb = torch.max(boxes1[:, 2:], boxes2[:, 2:]) + + wh = (rb - lt).clamp(min=0) # [N,2] + area = wh[:, 0] * wh[:, 1] + + return iou - (area - union) / area + +def masks_to_boxes(masks): + """Compute the bounding boxes around the provided masks + + The masks should be in format [N, H, W] where N is the number of masks, (H, W) are the spatial dimensions. + + Returns a [N, 4] tensors, with the boxes in xyxy format + """ + if masks.numel() == 0: + return torch.zeros((0, 4), device=masks.device) + + h, w = masks.shape[-2:] + + y = torch.arange(0, h, dtype=torch.float) + x = torch.arange(0, w, dtype=torch.float) + y, x = torch.meshgrid(y, x) + + x_mask = (masks * x.unsqueeze(0)) + x_max = x_mask.flatten(1).max(-1)[0] + x_min = x_mask.masked_fill(~(masks.bool()), 1e8).flatten(1).min(-1)[0] + + y_mask = (masks * y.unsqueeze(0)) + y_max = y_mask.flatten(1).max(-1)[0] + y_min = y_mask.masked_fill(~(masks.bool()), 1e8).flatten(1).min(-1)[0] + + return torch.stack([x_min, y_min, x_max, y_max], 1) + +if __name__ == '__main__': + x = torch.rand(5, 4) + y = torch.rand(3, 4) + iou, union = box_iou(x, y) + import ipdb; ipdb.set_trace() \ No newline at end of file diff --git a/util/coco_id2name.json b/util/coco_id2name.json new file mode 100644 index 0000000000000000000000000000000000000000..52ad2e96e7974593e13ee56cd3eff0d22c59f96d --- /dev/null +++ b/util/coco_id2name.json @@ -0,0 +1 @@ +{"1": "person", "2": "bicycle", "3": "car", "4": "motorcycle", "5": "airplane", "6": "bus", "7": "train", "8": "truck", "9": "boat", "10": "traffic light", "11": "fire hydrant", "13": "stop sign", "14": "parking meter", "15": "bench", "16": "bird", "17": "cat", "18": "dog", "19": "horse", "20": "sheep", "21": "cow", "22": "elephant", "23": "bear", "24": "zebra", "25": "giraffe", "27": "backpack", "28": "umbrella", "31": "handbag", "32": "tie", "33": "suitcase", "34": "frisbee", "35": "skis", "36": "snowboard", "37": "sports ball", "38": "kite", "39": "baseball bat", "40": "baseball glove", "41": "skateboard", "42": "surfboard", "43": "tennis racket", "44": "bottle", "46": "wine glass", "47": "cup", "48": "fork", "49": "knife", "50": "spoon", "51": "bowl", "52": "banana", "53": "apple", "54": "sandwich", "55": "orange", "56": "broccoli", "57": "carrot", "58": "hot dog", "59": "pizza", "60": "donut", "61": "cake", "62": "chair", "63": "couch", "64": "potted plant", "65": "bed", "67": "dining table", "70": "toilet", "72": "tv", "73": "laptop", "74": "mouse", "75": "remote", "76": "keyboard", "77": "cell phone", "78": "microwave", "79": "oven", "80": "toaster", "81": "sink", "82": "refrigerator", "84": "book", "85": "clock", "86": "vase", "87": "scissors", "88": "teddy bear", "89": "hair drier", "90": "toothbrush"} \ No newline at end of file diff --git a/util/get_param_dicts.py b/util/get_param_dicts.py new file mode 100644 index 0000000000000000000000000000000000000000..5f96351e74f98557932393904f019df03b2cec9f --- /dev/null +++ b/util/get_param_dicts.py @@ -0,0 +1,85 @@ +import json +import torch +import torch.nn as nn + + +def match_name_keywords(n: str, name_keywords: list): + out = False + for b in name_keywords: + if b in n: + out = True + break + return out + + +def get_param_dict(args, model_without_ddp: nn.Module): + try: + param_dict_type = args.param_dict_type + except: + param_dict_type = 'default' + assert param_dict_type in ['default', 'ddetr_in_mmdet', 'large_wd'] + + # by default + # import pdb;pdb.set_trace() + if param_dict_type == 'default': + param_dicts = [ + {"params": [p for n, p in model_without_ddp.named_parameters() if "backbone" not in n and p.requires_grad]}, + { + "params": [p for n, p in model_without_ddp.named_parameters() if "backbone" in n and p.requires_grad], + "lr": args.lr_backbone, + } + ] + return param_dicts + + if param_dict_type == 'ddetr_in_mmdet': + param_dicts = [ + { + "params": + [p for n, p in model_without_ddp.named_parameters() + if not match_name_keywords(n, args.lr_backbone_names) and not match_name_keywords(n, args.lr_linear_proj_names) and p.requires_grad], + "lr": args.lr, + }, + { + "params": [p for n, p in model_without_ddp.named_parameters() + if match_name_keywords(n, args.lr_backbone_names) and p.requires_grad], + "lr": args.lr_backbone, + }, + { + "params": [p for n, p in model_without_ddp.named_parameters() + if match_name_keywords(n, args.lr_linear_proj_names) and p.requires_grad], + "lr": args.lr_linear_proj_mult, + } + ] + return param_dicts + + if param_dict_type == 'large_wd': + param_dicts = [ + { + "params": + [p for n, p in model_without_ddp.named_parameters() + if not match_name_keywords(n, ['backbone']) and not match_name_keywords(n, ['norm', 'bias']) and p.requires_grad], + }, + { + "params": [p for n, p in model_without_ddp.named_parameters() + if match_name_keywords(n, ['backbone']) and match_name_keywords(n, ['norm', 'bias']) and p.requires_grad], + "lr": args.lr_backbone, + "weight_decay": 0.0, + }, + { + "params": [p for n, p in model_without_ddp.named_parameters() + if match_name_keywords(n, ['backbone']) and not match_name_keywords(n, ['norm', 'bias']) and p.requires_grad], + "lr": args.lr_backbone, + "weight_decay": args.weight_decay, + }, + { + "params": + [p for n, p in model_without_ddp.named_parameters() + if not match_name_keywords(n, ['backbone']) and match_name_keywords(n, ['norm', 'bias']) and p.requires_grad], + "lr": args.lr, + "weight_decay": 0.0, + } + ] + + # print("param_dicts: {}".format(param_dicts)) + + return param_dicts \ No newline at end of file diff --git a/util/get_tokenlizer.py b/util/get_tokenlizer.py new file mode 100644 index 0000000000000000000000000000000000000000..77873bcf4e098c194ef6182b8cbfdc2aa4e3d0ed --- /dev/null +++ b/util/get_tokenlizer.py @@ -0,0 +1,29 @@ +from transformers import AutoTokenizer, BertModel, BertTokenizer, RobertaModel, RobertaTokenizerFast +import os + +def get_tokenlizer(text_encoder_type): + if not isinstance(text_encoder_type, str): + # print("text_encoder_type is not a str") + if hasattr(text_encoder_type, "text_encoder_type"): + text_encoder_type = text_encoder_type.text_encoder_type + elif text_encoder_type.get("text_encoder_type", False): + text_encoder_type = text_encoder_type.get("text_encoder_type") + elif os.path.isdir(text_encoder_type) and os.path.exists(text_encoder_type): + pass + else: + raise ValueError( + "Unknown type of text_encoder_type: {}".format(type(text_encoder_type)) + ) + print("final text_encoder_type: {}".format(text_encoder_type)) + tokenizer = AutoTokenizer.from_pretrained(text_encoder_type) + print("load tokenizer done.") + return tokenizer + + +def get_pretrained_language_model(text_encoder_type): + if text_encoder_type == "bert-base-uncased" or (os.path.isdir(text_encoder_type) and os.path.exists(text_encoder_type)): + return BertModel.from_pretrained(text_encoder_type) + if text_encoder_type == "roberta-base": + return RobertaModel.from_pretrained(text_encoder_type) + + raise ValueError("Unknown text_encoder_type {}".format(text_encoder_type)) diff --git a/util/logger.py b/util/logger.py new file mode 100644 index 0000000000000000000000000000000000000000..99d2fd8ef20b2d1354b3b5d3e6d8b4987e5c8ce2 --- /dev/null +++ b/util/logger.py @@ -0,0 +1,74 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved +import functools +import logging +import os +import sys +import colorlog + + +# so that calling setup_logger multiple times won't add many handlers +@functools.lru_cache() +def setup_logger( + output=None, distributed_rank=0, *, color=True, name="imagenet", abbrev_name=None +): + """ + Initialize the detectron2 logger and set its verbosity level to "INFO". + + Args: + output (str): a file name or a directory to save log. If None, will not save log file. + If ends with ".txt" or ".log", assumed to be a file name. + Otherwise, logs will be saved to `output/log.txt`. + name (str): the root module name of this logger + + Returns: + logging.Logger: a logger + """ + logger = logging.getLogger(name) + logger.setLevel(logging.DEBUG) + formatter = colorlog.ColoredFormatter( + "%(log_color)s%(levelname)-8s%(reset)s %(log_color)s%(asctime)s | %(blue)s%(message)s", + datefmt=None, + reset=True, + log_colors={ + 'DEBUG': 'cyan', + 'INFO': 'green', + 'WARNING': 'yellow', + 'ERROR': 'red', + 'CRITICAL': 'red,bg_white' + } + ) + logger.propagate = False + + if abbrev_name is None: + abbrev_name = name + + # stdout logging: master only + if distributed_rank == 0: + ch = logging.StreamHandler(stream=sys.stdout) + ch.setLevel(logging.DEBUG) + ch.setFormatter(formatter) + logger.addHandler(ch) + + # file logging: all workers + if output is not None: + if output.endswith(".txt") or output.endswith(".log"): + filename = output + else: + filename = os.path.join(output, "log.txt") + if distributed_rank > 0: + filename = filename + f".rank{distributed_rank}" + os.makedirs(os.path.dirname(filename), exist_ok=True) + + fh = logging.StreamHandler(_cached_log_stream(filename)) + fh.setLevel(logging.DEBUG) + fh.setFormatter(formatter) + logger.addHandler(fh) + + return logger + + +# cache the opened file object, so that different calls to `setup_logger` +# with the same file name can safely write to the same file. +@functools.lru_cache(maxsize=None) +def _cached_log_stream(filename): + return open(filename, "a") diff --git a/util/misc.py b/util/misc.py new file mode 100644 index 0000000000000000000000000000000000000000..4a3d759dea39fe8ad01f04d2bb19842b995f53e3 --- /dev/null +++ b/util/misc.py @@ -0,0 +1,626 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved +""" +Misc functions, including distributed helpers. + +Mostly copy-paste from torchvision references. +""" +import os +import random +import subprocess +import time +from collections import OrderedDict, defaultdict, deque +import datetime +import pickle +from typing import Optional, List + +import json, time +import numpy as np +import torch +import torch.distributed as dist +from torch import Tensor + +import colorsys + +# needed due to empty tensor bug in pytorch and torchvision 0.5 +import torchvision +__torchvision_need_compat_flag = float(torchvision.__version__.split('.')[1]) < 7 +if __torchvision_need_compat_flag: + from torchvision.ops import _new_empty_tensor + from torchvision.ops.misc import _output_size + + +class SmoothedValue(object): + """Track a series of values and provide access to smoothed values over a + window or the global series average. + """ + + def __init__(self, window_size=20, fmt=None): + if fmt is None: + fmt = "{median:.4f} ({global_avg:.4f})" + self.deque = deque(maxlen=window_size) + self.total = 0.0 + self.count = 0 + self.fmt = fmt + + def update(self, value, n=1): + self.deque.append(value) + self.count += n + self.total += value * n + + def synchronize_between_processes(self): + """ + Warning: does not synchronize the deque! + """ + if not is_dist_avail_and_initialized(): + return + t = torch.tensor([self.count, self.total], dtype=torch.float64, device='cuda') + dist.barrier() + dist.all_reduce(t) + t = t.tolist() + self.count = int(t[0]) + self.total = t[1] + + @property + def median(self): + d = torch.tensor(list(self.deque)) + if d.shape[0] == 0: + return 0 + return d.median().item() + + @property + def avg(self): + d = torch.tensor(list(self.deque), dtype=torch.float32) + return d.mean().item() + + @property + def global_avg(self): + return self.total / self.count + + @property + def max(self): + return max(self.deque) + + @property + def value(self): + return self.deque[-1] + + def __str__(self): + return self.fmt.format( + median=self.median, + avg=self.avg, + global_avg=self.global_avg, + max=self.max, + value=self.value) + + +def all_gather(data): + """ + Run all_gather on arbitrary picklable data (not necessarily tensors) + Args: + data: any picklable object + Returns: + list[data]: list of data gathered from each rank + """ + world_size = get_world_size() + if world_size == 1: + return [data] + + # serialized to a Tensor + buffer = pickle.dumps(data) + storage = torch.ByteStorage.from_buffer(buffer) + tensor = torch.ByteTensor(storage).to("cuda") + + # obtain Tensor size of each rank + local_size = torch.tensor([tensor.numel()], device="cuda") + size_list = [torch.tensor([0], device="cuda") for _ in range(world_size)] + dist.all_gather(size_list, local_size) + size_list = [int(size.item()) for size in size_list] + max_size = max(size_list) + + # receiving Tensor from all ranks + # we pad the tensor because torch all_gather does not support + # gathering tensors of different shapes + tensor_list = [] + for _ in size_list: + tensor_list.append(torch.empty((max_size,), dtype=torch.uint8, device="cuda")) + if local_size != max_size: + padding = torch.empty(size=(max_size - local_size,), dtype=torch.uint8, device="cuda") + tensor = torch.cat((tensor, padding), dim=0) + dist.all_gather(tensor_list, tensor) + + data_list = [] + for size, tensor in zip(size_list, tensor_list): + buffer = tensor.cpu().numpy().tobytes()[:size] + data_list.append(pickle.loads(buffer)) + + return data_list + + +def reduce_dict(input_dict, average=True): + """ + Args: + input_dict (dict): all the values will be reduced + average (bool): whether to do average or sum + Reduce the values in the dictionary from all processes so that all processes + have the averaged results. Returns a dict with the same fields as + input_dict, after reduction. + """ + world_size = get_world_size() + if world_size < 2: + return input_dict + with torch.no_grad(): + names = [] + values = [] + # sort the keys so that they are consistent across processes + for k in sorted(input_dict.keys()): + names.append(k) + values.append(input_dict[k]) + values = torch.stack(values, dim=0) + dist.all_reduce(values) + if average: + values /= world_size + reduced_dict = {k: v for k, v in zip(names, values)} + return reduced_dict + + +class MetricLogger(object): + def __init__(self, delimiter="\t"): + self.meters = defaultdict(SmoothedValue) + self.delimiter = delimiter + + def update(self, **kwargs): + for k, v in kwargs.items(): + if isinstance(v, torch.Tensor): + v = v.item() + assert isinstance(v, (float, int)) + self.meters[k].update(v) + + def __getattr__(self, attr): + if attr in self.meters: + return self.meters[attr] + if attr in self.__dict__: + return self.__dict__[attr] + raise AttributeError("'{}' object has no attribute '{}'".format( + type(self).__name__, attr)) + + def __str__(self): + loss_str = [] + for name, meter in self.meters.items(): + # print(name, str(meter)) + # import ipdb;ipdb.set_trace() + if meter.count > 0: + loss_str.append( + "{}: {}".format(name, str(meter)) + ) + return self.delimiter.join(loss_str) + + def synchronize_between_processes(self): + for meter in self.meters.values(): + meter.synchronize_between_processes() + + def add_meter(self, name, meter): + self.meters[name] = meter + + def log_every(self, iterable, print_freq, header=None, logger=None): + if logger is None: + print_func = print + else: + print_func = logger.info + + i = 0 + if not header: + header = '' + start_time = time.time() + end = time.time() + iter_time = SmoothedValue(fmt='{avg:.4f}') + data_time = SmoothedValue(fmt='{avg:.4f}') + space_fmt = ':' + str(len(str(len(iterable)))) + 'd' + if torch.cuda.is_available(): + log_msg = self.delimiter.join([ + header, + '[{0' + space_fmt + '}/{1}]', + 'eta: {eta}', + '{meters}', + 'time: {time}', + 'data: {data}', + 'max mem: {memory:.0f}' + ]) + else: + log_msg = self.delimiter.join([ + header, + '[{0' + space_fmt + '}/{1}]', + 'eta: {eta}', + '{meters}', + 'time: {time}', + 'data: {data}' + ]) + MB = 1024.0 * 1024.0 + for obj in iterable: + data_time.update(time.time() - end) + yield obj + + iter_time.update(time.time() - end) + if i % print_freq == 0 or i == len(iterable) - 1: + eta_seconds = iter_time.global_avg * (len(iterable) - i) + eta_string = str(datetime.timedelta(seconds=int(eta_seconds))) + if torch.cuda.is_available(): + print_func(log_msg.format( + i, len(iterable), eta=eta_string, + meters=str(self), + time=str(iter_time), data=str(data_time), + memory=torch.cuda.max_memory_allocated() / MB)) + else: + print_func(log_msg.format( + i, len(iterable), eta=eta_string, + meters=str(self), + time=str(iter_time), data=str(data_time))) + i += 1 + end = time.time() + total_time = time.time() - start_time + total_time_str = str(datetime.timedelta(seconds=int(total_time))) + print_func('{} Total time: {} ({:.4f} s / it)'.format( + header, total_time_str, total_time / len(iterable))) + + +def get_sha(): + cwd = os.path.dirname(os.path.abspath(__file__)) + + def _run(command): + return subprocess.check_output(command, cwd=cwd).decode('ascii').strip() + sha = 'N/A' + diff = "clean" + branch = 'N/A' + try: + sha = _run(['git', 'rev-parse', 'HEAD']) + subprocess.check_output(['git', 'diff'], cwd=cwd) + diff = _run(['git', 'diff-index', 'HEAD']) + diff = "has uncommited changes" if diff else "clean" + branch = _run(['git', 'rev-parse', '--abbrev-ref', 'HEAD']) + except Exception: + pass + message = f"sha: {sha}, status: {diff}, branch: {branch}" + return message + + +def collate_fn(batch): + + batch = list(zip(*batch)) + batch[0] = nested_tensor_from_tensor_list(batch[0]) + return tuple(batch) + + +def _max_by_axis(the_list): + # type: (List[List[int]]) -> List[int] + maxes = the_list[0] + for sublist in the_list[1:]: + for index, item in enumerate(sublist): + maxes[index] = max(maxes[index], item) + return maxes + + +class NestedTensor(object): + def __init__(self, tensors, mask: Optional[Tensor]): + self.tensors = tensors + self.mask = mask + if mask == 'auto': + self.mask = torch.zeros_like(tensors).to(tensors.device) + if self.mask.dim() == 3: + self.mask = self.mask.sum(0).to(bool) + elif self.mask.dim() == 4: + self.mask = self.mask.sum(1).to(bool) + else: + raise ValueError("tensors dim must be 3 or 4 but {}({})".format(self.tensors.dim(), self.tensors.shape)) + + def imgsize(self): + res = [] + for i in range(self.tensors.shape[0]): + mask = self.mask[i] + maxH = (~mask).sum(0).max() + maxW = (~mask).sum(1).max() + res.append(torch.Tensor([maxH, maxW])) + return res + + def to(self, device): + # type: (Device) -> NestedTensor # noqa + cast_tensor = self.tensors.to(device) + mask = self.mask + if mask is not None: + assert mask is not None + cast_mask = mask.to(device) + else: + cast_mask = None + return NestedTensor(cast_tensor, cast_mask) + + def to_img_list_single(self, tensor, mask): + assert tensor.dim() == 3, "dim of tensor should be 3 but {}".format(tensor.dim()) + maxH = (~mask).sum(0).max() + maxW = (~mask).sum(1).max() + img = tensor[:, :maxH, :maxW] + return img + + def to_img_list(self): + """remove the padding and convert to img list + + Returns: + [type]: [description] + """ + if self.tensors.dim() == 3: + return self.to_img_list_single(self.tensors, self.mask) + else: + res = [] + for i in range(self.tensors.shape[0]): + tensor_i = self.tensors[i] + mask_i = self.mask[i] + res.append(self.to_img_list_single(tensor_i, mask_i)) + return res + + @property + def device(self): + return self.tensors.device + + def decompose(self): + return self.tensors, self.mask + + def __repr__(self): + return str(self.tensors) + + @property + def shape(self): + return { + 'tensors.shape': self.tensors.shape, + 'mask.shape': self.mask.shape + } + + +def nested_tensor_from_tensor_list(tensor_list: List[Tensor]): + # TODO make this more general + if tensor_list[0].ndim == 3: + if torchvision._is_tracing(): + # nested_tensor_from_tensor_list() does not export well to ONNX + # call _onnx_nested_tensor_from_tensor_list() instead + return _onnx_nested_tensor_from_tensor_list(tensor_list) + + # TODO make it support different-sized images + max_size = _max_by_axis([list(img.shape) for img in tensor_list]) + # min_size = tuple(min(s) for s in zip(*[img.shape for img in tensor_list])) + batch_shape = [len(tensor_list)] + max_size + b, c, h, w = batch_shape + dtype = tensor_list[0].dtype + device = tensor_list[0].device + tensor = torch.zeros(batch_shape, dtype=dtype, device=device) + mask = torch.ones((b, h, w), dtype=torch.bool, device=device) + for img, pad_img, m in zip(tensor_list, tensor, mask): + pad_img[: img.shape[0], : img.shape[1], : img.shape[2]].copy_(img) + m[: img.shape[1], :img.shape[2]] = False + else: + raise ValueError('not supported') + return NestedTensor(tensor, mask) + + +# _onnx_nested_tensor_from_tensor_list() is an implementation of +# nested_tensor_from_tensor_list() that is supported by ONNX tracing. +@torch.jit.unused +def _onnx_nested_tensor_from_tensor_list(tensor_list: List[Tensor]) -> NestedTensor: + max_size = [] + for i in range(tensor_list[0].dim()): + max_size_i = torch.max(torch.stack([img.shape[i] for img in tensor_list]).to(torch.float32)).to(torch.int64) + max_size.append(max_size_i) + max_size = tuple(max_size) + + # work around for + # pad_img[: img.shape[0], : img.shape[1], : img.shape[2]].copy_(img) + # m[: img.shape[1], :img.shape[2]] = False + # which is not yet supported in onnx + padded_imgs = [] + padded_masks = [] + for img in tensor_list: + padding = [(s1 - s2) for s1, s2 in zip(max_size, tuple(img.shape))] + padded_img = torch.nn.functional.pad(img, (0, padding[2], 0, padding[1], 0, padding[0])) + padded_imgs.append(padded_img) + + m = torch.zeros_like(img[0], dtype=torch.int, device=img.device) + padded_mask = torch.nn.functional.pad(m, (0, padding[2], 0, padding[1]), "constant", 1) + padded_masks.append(padded_mask.to(torch.bool)) + + tensor = torch.stack(padded_imgs) + mask = torch.stack(padded_masks) + + return NestedTensor(tensor, mask=mask) + + +def setup_for_distributed(is_master): + """ + This function disables printing when not in master process + """ + import builtins as __builtin__ + builtin_print = __builtin__.print + + def print(*args, **kwargs): + force = kwargs.pop('force', False) + if is_master or force: + builtin_print(*args, **kwargs) + + __builtin__.print = print + + +def is_dist_avail_and_initialized(): + if not dist.is_available(): + return False + if not dist.is_initialized(): + return False + return True + + +def get_world_size(): + if not is_dist_avail_and_initialized(): + return 1 + return dist.get_world_size() + + +def get_rank(): + if not is_dist_avail_and_initialized(): + return 0 + return dist.get_rank() + + +def is_main_process(): + return get_rank() == 0 + + +def save_on_master(*args, **kwargs): + if is_main_process(): + torch.save(*args, **kwargs) + + +def init_distributed_mode(args): + if 'WORLD_SIZE' in os.environ and os.environ['WORLD_SIZE'] != '': # 'RANK' in os.environ and + # args.rank = int(os.environ["RANK"]) + # args.world_size = int(os.environ['WORLD_SIZE']) + # args.gpu = args.local_rank = int(os.environ['LOCAL_RANK']) + + # launch by torch.distributed.launch + # Single node + # python -m torch.distributed.launch --nproc_per_node=8 main.py --world-size 1 --rank 0 ... + # Multi nodes + # python -m torch.distributed.launch --nproc_per_node=8 main.py --world-size 2 --rank 0 --dist-url 'tcp://IP_OF_NODE0:FREEPORT' ... + # python -m torch.distributed.launch --nproc_per_node=8 main.py --world-size 2 --rank 1 --dist-url 'tcp://IP_OF_NODE0:FREEPORT' ... + + local_world_size = int(os.environ['WORLD_SIZE']) + args.world_size = args.world_size * local_world_size + args.gpu = args.local_rank = int(os.environ['LOCAL_RANK']) + args.rank = args.rank * local_world_size + args.local_rank + print('world size: {}, rank: {}, local rank: {}'.format(args.world_size, args.rank, args.local_rank)) + print(json.dumps(dict(os.environ), indent=2)) + elif 'SLURM_PROCID' in os.environ: + args.rank = int(os.environ['SLURM_PROCID']) + args.gpu = args.local_rank = int(os.environ['SLURM_LOCALID']) + args.world_size = int(os.environ['SLURM_NPROCS']) + + print('world size: {}, world rank: {}, local rank: {}, device_count: {}'.format(args.world_size, args.rank, args.local_rank, torch.cuda.device_count())) + else: + print('Not using distributed mode') + args.distributed = False + args.world_size = 1 + args.rank = 0 + args.local_rank = 0 + return + + print("world_size:{} rank:{} local_rank:{}".format(args.world_size, args.rank, args.local_rank)) + args.distributed = True + torch.cuda.set_device(args.local_rank) + args.dist_backend = 'nccl' + print('| distributed init (rank {}): {}'.format(args.rank, args.dist_url), flush=True) + torch.distributed.init_process_group(backend=args.dist_backend, init_method=args.dist_url, + world_size=args.world_size, rank=args.rank) + print("Before torch.distributed.barrier()") + torch.distributed.barrier() + print("End torch.distributed.barrier()") + setup_for_distributed(args.rank == 0) + +def setup_distributed(args): + if 'WORLD_SIZE' in os.environ and os.environ['WORLD_SIZE'] != '': # 'RANK' in os.environ and + local_world_size = int(os.environ['WORLD_SIZE']) + args.world_size = args.world_size * local_world_size + args.gpu = args.local_rank = int(os.environ['LOCAL_RANK']) + args.rank = args.rank * local_world_size + args.local_rank + print('world size: {}, rank: {}, local rank: {}'.format(args.world_size, args.rank, args.local_rank)) + print(json.dumps(dict(os.environ), indent=2)) + elif 'SLURM_PROCID' in os.environ: + args.rank = int(os.environ['SLURM_PROCID']) + args.gpu = args.local_rank = int(os.environ['SLURM_LOCALID']) + args.world_size = int(os.environ['SLURM_NTASKS']) + node_list = os.environ["SLURM_NODELIST"] + addr = subprocess.getoutput(f"scontrol show hostname {node_list} | head -n1") + if "MASTER_PORT" not in os.environ: + os.environ["MASTER_PORT"] = "23233" + if "MASTER_ADDR" not in os.environ: + os.environ["MASTER_ADDR"] = addr + os.environ["WORLD_SIZE"] = str(args.world_size) + os.environ["LOCAL_RANK"] = str(args.local_rank) + os.environ["RANK"] = str(args.rank) + print('world size: {}, world rank: {}, local rank: {}, device_count: {}'.format(args.world_size, args.rank, args.local_rank, torch.cuda.device_count())) + else: + print('Not using distributed mode') + args.distributed = False + args.world_size = 1 + args.rank = 0 + args.local_rank = 0 + return + + args.distributed = True + torch.cuda.set_device(args.local_rank) + args.dist_backend = 'nccl' + torch.distributed.init_process_group(backend=args.dist_backend, init_method=args.dist_url, + world_size=args.world_size, rank=args.rank) + torch.distributed.barrier() + print(f' == distributed init (rank {args.rank}) done.') + + setup_for_distributed(args.rank == 0) + +@torch.no_grad() +def accuracy(output, target, topk=(1,)): + """Computes the precision@k for the specified values of k""" + if target.numel() == 0: + return [torch.zeros([], device=output.device)] + maxk = max(topk) + batch_size = target.size(0) + + _, pred = output.topk(maxk, 1, True, True) + pred = pred.t() + correct = pred.eq(target.view(1, -1).expand_as(pred)) + + res = [] + for k in topk: + correct_k = correct[:k].view(-1).float().sum(0) + res.append(correct_k.mul_(100.0 / batch_size)) + return res + + +def interpolate(input, size=None, scale_factor=None, mode="nearest", align_corners=None): + # type: (Tensor, Optional[List[int]], Optional[float], str, Optional[bool]) -> Tensor + """ + Equivalent to nn.functional.interpolate, but with support for empty batch sizes. + This will eventually be supported natively by PyTorch, and this + class can go away. + """ + if __torchvision_need_compat_flag < 0.7: + if input.numel() > 0: + return torch.nn.functional.interpolate( + input, size, scale_factor, mode, align_corners + ) + + output_shape = _output_size(2, input, size, scale_factor) + output_shape = list(input.shape[:-2]) + list(output_shape) + return _new_empty_tensor(input, output_shape) + else: + return torchvision.ops.misc.interpolate(input, size, scale_factor, mode, align_corners) + + + +class color_sys(): + def __init__(self, num_colors) -> None: + self.num_colors = num_colors + colors=[] + for i in np.arange(0., 360., 360. / num_colors): + hue = i/360. + lightness = (50 + np.random.rand() * 10)/100. + saturation = (90 + np.random.rand() * 10)/100. + colors.append(tuple([int(j*255) for j in colorsys.hls_to_rgb(hue, lightness, saturation)])) + self.colors = colors + + def __call__(self, idx): + return self.colors[idx] + +def inverse_sigmoid(x, eps=1e-3): + x = x.clamp(min=0, max=1) + x1 = x.clamp(min=eps) + x2 = (1 - x).clamp(min=eps) + return torch.log(x1/x2) + +def clean_state_dict(state_dict): + new_state_dict = OrderedDict() + for k, v in state_dict.items(): + if k[:7] == 'module.': + k = k[7:] # remove `module.` + new_state_dict[k] = v + return new_state_dict \ No newline at end of file diff --git a/util/plot_utils.py b/util/plot_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..af67acd9f7a3ad7ec61908adfa2c67acb734ca08 --- /dev/null +++ b/util/plot_utils.py @@ -0,0 +1,112 @@ +""" +Plotting utilities to visualize training logs. +""" +import torch +import pandas as pd +import numpy as np +import seaborn as sns +import matplotlib.pyplot as plt + +from pathlib import Path, PurePath + + +def plot_logs(logs, fields=('class_error', 'loss_bbox_unscaled', 'mAP'), ewm_col=0, log_name='log.txt'): + ''' + Function to plot specific fields from training log(s). Plots both training and test results. + + :: Inputs - logs = list containing Path objects, each pointing to individual dir with a log file + - fields = which results to plot from each log file - plots both training and test for each field. + - ewm_col = optional, which column to use as the exponential weighted smoothing of the plots + - log_name = optional, name of log file if different than default 'log.txt'. + + :: Outputs - matplotlib plots of results in fields, color coded for each log file. + - solid lines are training results, dashed lines are test results. + + ''' + func_name = "plot_utils.py::plot_logs" + + # verify logs is a list of Paths (list[Paths]) or single Pathlib object Path, + # convert single Path to list to avoid 'not iterable' error + + if not isinstance(logs, list): + if isinstance(logs, PurePath): + logs = [logs] + print(f"{func_name} info: logs param expects a list argument, converted to list[Path].") + else: + raise ValueError(f"{func_name} - invalid argument for logs parameter.\n \ + Expect list[Path] or single Path obj, received {type(logs)}") + + # Quality checks - verify valid dir(s), that every item in list is Path object, and that log_name exists in each dir + for i, dir in enumerate(logs): + if not isinstance(dir, PurePath): + raise ValueError(f"{func_name} - non-Path object in logs argument of {type(dir)}: \n{dir}") + if not dir.exists(): + raise ValueError(f"{func_name} - invalid directory in logs argument:\n{dir}") + # verify log_name exists + fn = Path(dir / log_name) + if not fn.exists(): + print(f"-> missing {log_name}. Have you gotten to Epoch 1 in training?") + print(f"--> full path of missing log file: {fn}") + return + + # load log file(s) and plot + dfs = [pd.read_json(Path(p) / log_name, lines=True) for p in logs] + + fig, axs = plt.subplots(ncols=len(fields), figsize=(16, 5)) + + for df, color in zip(dfs, sns.color_palette(n_colors=len(logs))): + for j, field in enumerate(fields): + if field == 'mAP': + coco_eval = pd.DataFrame( + np.stack(df.test_coco_eval_bbox.dropna().values)[:, 1] + ).ewm(com=ewm_col).mean() + axs[j].plot(coco_eval, c=color) + else: + df.interpolate().ewm(com=ewm_col).mean().plot( + y=[f'train_{field}', f'test_{field}'], + ax=axs[j], + color=[color] * 2, + style=['-', '--'] + ) + for ax, field in zip(axs, fields): + if field == 'mAP': + ax.legend([Path(p).name for p in logs]) + ax.set_title(field) + else: + ax.legend([f'train', f'test']) + ax.set_title(field) + + return fig, axs + +def plot_precision_recall(files, naming_scheme='iter'): + if naming_scheme == 'exp_id': + # name becomes exp_id + names = [f.parts[-3] for f in files] + elif naming_scheme == 'iter': + names = [f.stem for f in files] + else: + raise ValueError(f'not supported {naming_scheme}') + fig, axs = plt.subplots(ncols=2, figsize=(16, 5)) + for f, color, name in zip(files, sns.color_palette("Blues", n_colors=len(files)), names): + data = torch.load(f) + # precision is n_iou, n_points, n_cat, n_area, max_det + precision = data['precision'] + recall = data['params'].recThrs + scores = data['scores'] + # take precision for all classes, all areas and 100 detections + precision = precision[0, :, :, 0, -1].mean(1) + scores = scores[0, :, :, 0, -1].mean(1) + prec = precision.mean() + rec = data['recall'][0, :, 0, -1].mean() + print(f'{naming_scheme} {name}: mAP@50={prec * 100: 05.1f}, ' + + f'score={scores.mean():0.3f}, ' + + f'f1={2 * prec * rec / (prec + rec + 1e-8):0.3f}' + ) + axs[0].plot(recall, precision, c=color) + axs[1].plot(recall, scores, c=color) + + axs[0].set_title('Precision / Recall') + axs[0].legend(names) + axs[1].set_title('Scores / Recall') + axs[1].legend(names) + return fig, axs diff --git a/util/slconfig.py b/util/slconfig.py new file mode 100644 index 0000000000000000000000000000000000000000..8064727596d57ccbe8e9f94dbb0bba9be732812a --- /dev/null +++ b/util/slconfig.py @@ -0,0 +1,440 @@ +# ========================================================== +# Modified from mmcv +# ========================================================== +import os, sys +import os.path as osp +import ast +import tempfile +import shutil +from importlib import import_module + +from argparse import Action + +from addict import Dict +from yapf.yapflib.yapf_api import FormatCode + +import platform +MACOS, LINUX, WINDOWS = (platform.system() == x for x in ['Darwin', 'Linux', 'Windows']) # environment booleans + +BASE_KEY = '_base_' +DELETE_KEY = '_delete_' +RESERVED_KEYS = ['filename', 'text', 'pretty_text', 'get', 'dump', 'merge_from_dict'] + + +def check_file_exist(filename, msg_tmpl='file "{}" does not exist'): + if not osp.isfile(filename): + raise FileNotFoundError(msg_tmpl.format(filename)) + +class ConfigDict(Dict): + + def __missing__(self, name): + raise KeyError(name) + + def __getattr__(self, name): + try: + value = super(ConfigDict, self).__getattr__(name) + except KeyError: + ex = AttributeError(f"'{self.__class__.__name__}' object has no " + f"attribute '{name}'") + except Exception as e: + ex = e + else: + return value + raise ex + + +class SLConfig(object): + """ + config files. + only support .py file as config now. + + ref: mmcv.utils.config + + Example: + >>> cfg = Config(dict(a=1, b=dict(b1=[0, 1]))) + >>> cfg.a + 1 + >>> cfg.b + {'b1': [0, 1]} + >>> cfg.b.b1 + [0, 1] + >>> cfg = Config.fromfile('tests/data/config/a.py') + >>> cfg.filename + "/home/kchen/projects/mmcv/tests/data/config/a.py" + >>> cfg.item4 + 'test' + >>> cfg + "Config [path: /home/kchen/projects/mmcv/tests/data/config/a.py]: " + "{'item1': [1, 2], 'item2': {'a': 0}, 'item3': True, 'item4': 'test'}" + """ + @staticmethod + def _validate_py_syntax(filename): + with open(filename) as f: + content = f.read() + try: + ast.parse(content) + except SyntaxError: + raise SyntaxError('There are syntax errors in config ' + f'file {filename}') + + @staticmethod + def _file2dict(filename): + filename = osp.abspath(osp.expanduser(filename)) + check_file_exist(filename) + if filename.lower().endswith('.py'): + with tempfile.TemporaryDirectory() as temp_config_dir: + temp_config_file = tempfile.NamedTemporaryFile( + dir=temp_config_dir, suffix='.py') + temp_config_name = osp.basename(temp_config_file.name) + if WINDOWS: + temp_config_file.close() + shutil.copyfile(filename, + osp.join(temp_config_dir, temp_config_name)) + temp_module_name = osp.splitext(temp_config_name)[0] + sys.path.insert(0, temp_config_dir) + SLConfig._validate_py_syntax(filename) + mod = import_module(temp_module_name) + sys.path.pop(0) + cfg_dict = { + name: value + for name, value in mod.__dict__.items() + if not name.startswith('__') + } + # delete imported module + del sys.modules[temp_module_name] + # close temp file + temp_config_file.close() + elif filename.lower().endswith(('.yml', '.yaml', '.json')): + from .slio import slload + cfg_dict = slload(filename) + else: + raise IOError('Only py/yml/yaml/json type are supported now!') + + cfg_text = filename + '\n' + with open(filename, 'r') as f: + cfg_text += f.read() + + # parse the base file + if BASE_KEY in cfg_dict: + cfg_dir = osp.dirname(filename) + base_filename = cfg_dict.pop(BASE_KEY) + base_filename = base_filename if isinstance( + base_filename, list) else [base_filename] + + cfg_dict_list = list() + cfg_text_list = list() + for f in base_filename: + _cfg_dict, _cfg_text = SLConfig._file2dict(osp.join(cfg_dir, f)) + cfg_dict_list.append(_cfg_dict) + cfg_text_list.append(_cfg_text) + + base_cfg_dict = dict() + for c in cfg_dict_list: + if len(base_cfg_dict.keys() & c.keys()) > 0: + raise KeyError('Duplicate key is not allowed among bases') + # TODO Allow the duplicate key while warnning user + base_cfg_dict.update(c) + + base_cfg_dict = SLConfig._merge_a_into_b(cfg_dict, base_cfg_dict) + cfg_dict = base_cfg_dict + + # merge cfg_text + cfg_text_list.append(cfg_text) + cfg_text = '\n'.join(cfg_text_list) + + return cfg_dict, cfg_text + + @staticmethod + def _merge_a_into_b(a, b): + """merge dict `a` into dict `b` (non-inplace). + values in `a` will overwrite `b`. + copy first to avoid inplace modification + + Args: + a ([type]): [description] + b ([type]): [description] + + Returns: + [dict]: [description] + """ + + if not isinstance(a, dict): + return a + + b = b.copy() + for k, v in a.items(): + if isinstance(v, dict) and k in b and not v.pop(DELETE_KEY, False): + + if not isinstance(b[k], dict) and not isinstance(b[k], list): + # if : + + raise TypeError( + f'{k}={v} in child config cannot inherit from base ' + f'because {k} is a dict in the child config but is of ' + f'type {type(b[k])} in base config. You may set ' + f'`{DELETE_KEY}=True` to ignore the base config') + b[k] = SLConfig._merge_a_into_b(v, b[k]) + elif isinstance(b, list): + try: + _ = int(k) + except: + raise TypeError( + f'b is a list, ' + f'index {k} should be an int when input but {type(k)}' + ) + b[int(k)] = SLConfig._merge_a_into_b(v, b[int(k)]) + else: + b[k] = v + + return b + + @staticmethod + def fromfile(filename): + cfg_dict, cfg_text = SLConfig._file2dict(filename) + return SLConfig(cfg_dict, cfg_text=cfg_text, filename=filename) + + + def __init__(self, cfg_dict=None, cfg_text=None, filename=None): + if cfg_dict is None: + cfg_dict = dict() + elif not isinstance(cfg_dict, dict): + raise TypeError('cfg_dict must be a dict, but ' + f'got {type(cfg_dict)}') + for key in cfg_dict: + if key in RESERVED_KEYS: + raise KeyError(f'{key} is reserved for config file') + + super(SLConfig, self).__setattr__('_cfg_dict', ConfigDict(cfg_dict)) + super(SLConfig, self).__setattr__('_filename', filename) + if cfg_text: + text = cfg_text + elif filename: + with open(filename, 'r') as f: + text = f.read() + else: + text = '' + super(SLConfig, self).__setattr__('_text', text) + + + @property + def filename(self): + return self._filename + + @property + def text(self): + return self._text + + @property + def pretty_text(self): + + indent = 4 + + def _indent(s_, num_spaces): + s = s_.split('\n') + if len(s) == 1: + return s_ + first = s.pop(0) + s = [(num_spaces * ' ') + line for line in s] + s = '\n'.join(s) + s = first + '\n' + s + return s + + def _format_basic_types(k, v, use_mapping=False): + if isinstance(v, str): + v_str = f"'{v}'" + else: + v_str = str(v) + + if use_mapping: + k_str = f"'{k}'" if isinstance(k, str) else str(k) + attr_str = f'{k_str}: {v_str}' + else: + attr_str = f'{str(k)}={v_str}' + attr_str = _indent(attr_str, indent) + + return attr_str + + def _format_list(k, v, use_mapping=False): + # check if all items in the list are dict + if all(isinstance(_, dict) for _ in v): + v_str = '[\n' + v_str += '\n'.join( + f'dict({_indent(_format_dict(v_), indent)}),' + for v_ in v).rstrip(',') + if use_mapping: + k_str = f"'{k}'" if isinstance(k, str) else str(k) + attr_str = f'{k_str}: {v_str}' + else: + attr_str = f'{str(k)}={v_str}' + attr_str = _indent(attr_str, indent) + ']' + else: + attr_str = _format_basic_types(k, v, use_mapping) + return attr_str + + def _contain_invalid_identifier(dict_str): + contain_invalid_identifier = False + for key_name in dict_str: + contain_invalid_identifier |= \ + (not str(key_name).isidentifier()) + return contain_invalid_identifier + + def _format_dict(input_dict, outest_level=False): + r = '' + s = [] + + use_mapping = _contain_invalid_identifier(input_dict) + if use_mapping: + r += '{' + for idx, (k, v) in enumerate(input_dict.items()): + is_last = idx >= len(input_dict) - 1 + end = '' if outest_level or is_last else ',' + if isinstance(v, dict): + v_str = '\n' + _format_dict(v) + if use_mapping: + k_str = f"'{k}'" if isinstance(k, str) else str(k) + attr_str = f'{k_str}: dict({v_str}' + else: + attr_str = f'{str(k)}=dict({v_str}' + attr_str = _indent(attr_str, indent) + ')' + end + elif isinstance(v, list): + attr_str = _format_list(k, v, use_mapping) + end + else: + attr_str = _format_basic_types(k, v, use_mapping) + end + + s.append(attr_str) + r += '\n'.join(s) + if use_mapping: + r += '}' + return r + + cfg_dict = self._cfg_dict.to_dict() + text = _format_dict(cfg_dict, outest_level=True) + # copied from setup.cfg + yapf_style = dict( + based_on_style='pep8', + blank_line_before_nested_class_or_def=True, + split_before_expression_after_opening_paren=True) + text, _ = FormatCode(text, style_config=yapf_style, verify=True) + + return text + + + def __repr__(self): + return f'Config (path: {self.filename}): {self._cfg_dict.__repr__()}' + + def __len__(self): + return len(self._cfg_dict) + + def __getattr__(self, name): + # # debug + # print('+'*15) + # print('name=%s' % name) + # print("addr:", id(self)) + # # print('type(self):', type(self)) + # print(self.__dict__) + # print('+'*15) + # if self.__dict__ == {}: + # raise ValueError + + return getattr(self._cfg_dict, name) + + def __getitem__(self, name): + return self._cfg_dict.__getitem__(name) + + def __setattr__(self, name, value): + if isinstance(value, dict): + value = ConfigDict(value) + self._cfg_dict.__setattr__(name, value) + + def __setitem__(self, name, value): + if isinstance(value, dict): + value = ConfigDict(value) + self._cfg_dict.__setitem__(name, value) + + def __iter__(self): + return iter(self._cfg_dict) + + def dump(self, file=None): + + if file is None: + return self.pretty_text + else: + with open(file, 'w') as f: + f.write(self.pretty_text) + + def merge_from_dict(self, options): + """Merge list into cfg_dict + + Merge the dict parsed by MultipleKVAction into this cfg. + + Examples: + >>> options = {'model.backbone.depth': 50, + ... 'model.backbone.with_cp':True} + >>> cfg = Config(dict(model=dict(backbone=dict(type='ResNet')))) + >>> cfg.merge_from_dict(options) + >>> cfg_dict = super(Config, self).__getattribute__('_cfg_dict') + >>> assert cfg_dict == dict( + ... model=dict(backbone=dict(depth=50, with_cp=True))) + + Args: + options (dict): dict of configs to merge from. + """ + option_cfg_dict = {} + for full_key, v in options.items(): + d = option_cfg_dict + key_list = full_key.split('.') + for subkey in key_list[:-1]: + d.setdefault(subkey, ConfigDict()) + d = d[subkey] + subkey = key_list[-1] + d[subkey] = v + + cfg_dict = super(SLConfig, self).__getattribute__('_cfg_dict') + super(SLConfig, self).__setattr__( + '_cfg_dict', SLConfig._merge_a_into_b(option_cfg_dict, cfg_dict)) + + # for multiprocess + def __setstate__(self, state): + self.__init__(state) + + + def copy(self): + return SLConfig(self._cfg_dict.copy()) + + def deepcopy(self): + return SLConfig(self._cfg_dict.deepcopy()) + + +class DictAction(Action): + """ + argparse action to split an argument into KEY=VALUE form + on the first = and append to a dictionary. List options should + be passed as comma separated values, i.e KEY=V1,V2,V3 + """ + + @staticmethod + def _parse_int_float_bool(val): + try: + return int(val) + except ValueError: + pass + try: + return float(val) + except ValueError: + pass + if val.lower() in ['true', 'false']: + return True if val.lower() == 'true' else False + if val.lower() in ['none', 'null']: + return None + return val + + def __call__(self, parser, namespace, values, option_string=None): + options = {} + for kv in values: + key, val = kv.split('=', maxsplit=1) + val = [self._parse_int_float_bool(v) for v in val.split(',')] + if len(val) == 1: + val = val[0] + options[key] = val + setattr(namespace, self.dest, options) + diff --git a/util/slio.py b/util/slio.py new file mode 100644 index 0000000000000000000000000000000000000000..8b8f4dad2441b8352ab7311dbf16019515441331 --- /dev/null +++ b/util/slio.py @@ -0,0 +1,173 @@ +# ========================================================== +# Modified from mmcv +# ========================================================== + +import json, pickle, yaml +try: + from yaml import CLoader as Loader, CDumper as Dumper +except ImportError: + from yaml import Loader, Dumper + +from pathlib import Path +from abc import ABCMeta, abstractmethod + +# =========================== +# Rigister handler +# =========================== + +class BaseFileHandler(metaclass=ABCMeta): + + @abstractmethod + def load_from_fileobj(self, file, **kwargs): + pass + + @abstractmethod + def dump_to_fileobj(self, obj, file, **kwargs): + pass + + @abstractmethod + def dump_to_str(self, obj, **kwargs): + pass + + def load_from_path(self, filepath, mode='r', **kwargs): + with open(filepath, mode) as f: + return self.load_from_fileobj(f, **kwargs) + + def dump_to_path(self, obj, filepath, mode='w', **kwargs): + with open(filepath, mode) as f: + self.dump_to_fileobj(obj, f, **kwargs) + +class JsonHandler(BaseFileHandler): + + def load_from_fileobj(self, file): + return json.load(file) + + def dump_to_fileobj(self, obj, file, **kwargs): + json.dump(obj, file, **kwargs) + + def dump_to_str(self, obj, **kwargs): + return json.dumps(obj, **kwargs) + +class PickleHandler(BaseFileHandler): + + def load_from_fileobj(self, file, **kwargs): + return pickle.load(file, **kwargs) + + def load_from_path(self, filepath, **kwargs): + return super(PickleHandler, self).load_from_path( + filepath, mode='rb', **kwargs) + + def dump_to_str(self, obj, **kwargs): + kwargs.setdefault('protocol', 2) + return pickle.dumps(obj, **kwargs) + + def dump_to_fileobj(self, obj, file, **kwargs): + kwargs.setdefault('protocol', 2) + pickle.dump(obj, file, **kwargs) + + def dump_to_path(self, obj, filepath, **kwargs): + super(PickleHandler, self).dump_to_path( + obj, filepath, mode='wb', **kwargs) + +class YamlHandler(BaseFileHandler): + + def load_from_fileobj(self, file, **kwargs): + kwargs.setdefault('Loader', Loader) + return yaml.load(file, **kwargs) + + def dump_to_fileobj(self, obj, file, **kwargs): + kwargs.setdefault('Dumper', Dumper) + yaml.dump(obj, file, **kwargs) + + def dump_to_str(self, obj, **kwargs): + kwargs.setdefault('Dumper', Dumper) + return yaml.dump(obj, **kwargs) + +file_handlers = { + 'json': JsonHandler(), + 'yaml': YamlHandler(), + 'yml': YamlHandler(), + 'pickle': PickleHandler(), + 'pkl': PickleHandler() +} + +# =========================== +# load and dump +# =========================== + +def is_str(x): + """Whether the input is an string instance. + + Note: This method is deprecated since python 2 is no longer supported. + """ + return isinstance(x, str) + +def slload(file, file_format=None, **kwargs): + """Load data from json/yaml/pickle files. + + This method provides a unified api for loading data from serialized files. + + Args: + file (str or :obj:`Path` or file-like object): Filename or a file-like + object. + file_format (str, optional): If not specified, the file format will be + inferred from the file extension, otherwise use the specified one. + Currently supported formats include "json", "yaml/yml" and + "pickle/pkl". + + Returns: + The content from the file. + """ + if isinstance(file, Path): + file = str(file) + if file_format is None and is_str(file): + file_format = file.split('.')[-1] + if file_format not in file_handlers: + raise TypeError(f'Unsupported format: {file_format}') + + handler = file_handlers[file_format] + if is_str(file): + obj = handler.load_from_path(file, **kwargs) + elif hasattr(file, 'read'): + obj = handler.load_from_fileobj(file, **kwargs) + else: + raise TypeError('"file" must be a filepath str or a file-object') + return obj + + +def sldump(obj, file=None, file_format=None, **kwargs): + """Dump data to json/yaml/pickle strings or files. + + This method provides a unified api for dumping data as strings or to files, + and also supports custom arguments for each file format. + + Args: + obj (any): The python object to be dumped. + file (str or :obj:`Path` or file-like object, optional): If not + specified, then the object is dump to a str, otherwise to a file + specified by the filename or file-like object. + file_format (str, optional): Same as :func:`load`. + + Returns: + bool: True for success, False otherwise. + """ + if isinstance(file, Path): + file = str(file) + if file_format is None: + if is_str(file): + file_format = file.split('.')[-1] + elif file is None: + raise ValueError( + 'file_format must be specified since file is None') + if file_format not in file_handlers: + raise TypeError(f'Unsupported format: {file_format}') + + handler = file_handlers[file_format] + if file is None: + return handler.dump_to_str(obj, **kwargs) + elif is_str(file): + handler.dump_to_path(obj, file, **kwargs) + elif hasattr(file, 'write'): + handler.dump_to_fileobj(obj, file, **kwargs) + else: + raise TypeError('"file" must be a filename str or a file-object') diff --git a/util/static_data_path.py b/util/static_data_path.py new file mode 100644 index 0000000000000000000000000000000000000000..88a075a848cb634113563b1bcaff6d7c5f6982e2 --- /dev/null +++ b/util/static_data_path.py @@ -0,0 +1,10 @@ +coco = dict( + train = dict( + img_folder = '/comp_robot/cv_public_dataset/COCO2017/val2017', + ann_file = '/comp_robot/cv_public_dataset/COCO2017/annotations/instances_val2017.json' + ), + val = dict( + img_folder = '/comp_robot/cv_public_dataset/COCO2017/val2017', + ann_file = '/comp_robot/cv_public_dataset/COCO2017/annotations/instances_val2017.json' + ) +) \ No newline at end of file diff --git a/util/time_counter.py b/util/time_counter.py new file mode 100644 index 0000000000000000000000000000000000000000..19dc2e640bb0f19e686a5078d8e4d7db7ddaad96 --- /dev/null +++ b/util/time_counter.py @@ -0,0 +1,60 @@ +import json +import time + +class TimeCounter: + def __init__(self) -> None: + pass + + def clear(self): + self.timedict = {} + self.basetime = time.perf_counter() + + def timeit(self, name): + nowtime = time.perf_counter() - self.basetime + self.timedict[name] = nowtime + self.basetime = time.perf_counter() + + +class TimeHolder: + def __init__(self) -> None: + self.timedict = {} + + def update(self, _timedict:dict): + for k,v in _timedict.items(): + if k not in self.timedict: + self.timedict[k] = AverageMeter(name=k, val_only=True) + self.timedict[k].update(val=v) + + def final_res(self): + return {k:v.avg for k,v in self.timedict.items()} + + def __str__(self): + return json.dumps(self.final_res(), indent=2) + + +class AverageMeter(object): + """Computes and stores the average and current value""" + def __init__(self, name, fmt=':f', val_only=False): + self.name = name + self.fmt = fmt + self.val_only = val_only + self.reset() + + def reset(self): + self.val = 0 + self.avg = 0 + self.sum = 0 + self.count = 0 + + def update(self, val, n=1): + self.val = val + self.sum += val * n + self.count += n + self.avg = self.sum / self.count + + def __str__(self): + if self.val_only: + fmtstr = '{name} {val' + self.fmt + '}' + else: + fmtstr = '{name} {val' + self.fmt + '} ({avg' + self.fmt + '})' + return fmtstr.format(**self.__dict__) \ No newline at end of file diff --git a/util/utils.py b/util/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..8852c785f85969fd635e64854499ff731e82da1e --- /dev/null +++ b/util/utils.py @@ -0,0 +1,471 @@ +from collections import OrderedDict +from copy import deepcopy +import json +import warnings + +import torch +import numpy as np + +def slprint(x, name='x'): + if isinstance(x, (torch.Tensor, np.ndarray)): + print(f'{name}.shape:', x.shape) + elif isinstance(x, (tuple, list)): + print('type x:', type(x)) + for i in range(min(10, len(x))): + slprint(x[i], f'{name}[{i}]') + elif isinstance(x, dict): + for k,v in x.items(): + slprint(v, f'{name}[{k}]') + else: + print(f'{name}.type:', type(x)) + +def clean_state_dict(state_dict): + new_state_dict = OrderedDict() + for k, v in state_dict.items(): + if k[:7] == 'module.': + k = k[7:] # remove `module.` + new_state_dict[k] = v + return new_state_dict + +def renorm(img: torch.FloatTensor, mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) \ + -> torch.FloatTensor: + # img: tensor(3,H,W) or tensor(B,3,H,W) + # return: same as img + assert img.dim() == 3 or img.dim() == 4, "img.dim() should be 3 or 4 but %d" % img.dim() + if img.dim() == 3: + assert img.size(0) == 3, 'img.size(0) shoule be 3 but "%d". (%s)' % (img.size(0), str(img.size())) + img_perm = img.permute(1,2,0) + mean = torch.Tensor(mean) + std = torch.Tensor(std) + img_res = img_perm * std + mean + return img_res.permute(2,0,1) + else: # img.dim() == 4 + assert img.size(1) == 3, 'img.size(1) shoule be 3 but "%d". (%s)' % (img.size(1), str(img.size())) + img_perm = img.permute(0,2,3,1) + mean = torch.Tensor(mean) + std = torch.Tensor(std) + img_res = img_perm * std + mean + return img_res.permute(0,3,1,2) + + + +class CocoClassMapper(): + def __init__(self) -> None: + self.category_map_str = {"1": 1, "2": 2, "3": 3, "4": 4, "5": 5, "6": 6, "7": 7, "8": 8, "9": 9, "10": 10, "11": 11, "13": 12, "14": 13, "15": 14, "16": 15, "17": 16, "18": 17, "19": 18, "20": 19, "21": 20, "22": 21, "23": 22, "24": 23, "25": 24, "27": 25, "28": 26, "31": 27, "32": 28, "33": 29, "34": 30, "35": 31, "36": 32, "37": 33, "38": 34, "39": 35, "40": 36, "41": 37, "42": 38, "43": 39, "44": 40, "46": 41, "47": 42, "48": 43, "49": 44, "50": 45, "51": 46, "52": 47, "53": 48, "54": 49, "55": 50, "56": 51, "57": 52, "58": 53, "59": 54, "60": 55, "61": 56, "62": 57, "63": 58, "64": 59, "65": 60, "67": 61, "70": 62, "72": 63, "73": 64, "74": 65, "75": 66, "76": 67, "77": 68, "78": 69, "79": 70, "80": 71, "81": 72, "82": 73, "84": 74, "85": 75, "86": 76, "87": 77, "88": 78, "89": 79, "90": 80} + self.origin2compact_mapper = {int(k):v-1 for k,v in self.category_map_str.items()} + self.compact2origin_mapper = {int(v-1):int(k) for k,v in self.category_map_str.items()} + + def origin2compact(self, idx): + return self.origin2compact_mapper[int(idx)] + + def compact2origin(self, idx): + return self.compact2origin_mapper[int(idx)] + +def to_device(item, device): + if isinstance(item, torch.Tensor): + return item.to(device) + elif isinstance(item, list): + return [to_device(i, device) for i in item] + elif isinstance(item, dict): + return {k: to_device(v, device) for k,v in item.items()} + else: + raise NotImplementedError("Call Shilong if you use other containers! type: {}".format(type(item))) + + + +# +def get_gaussian_mean(x, axis, other_axis, softmax=True): + """ + + Args: + x (float): Input images(BxCxHxW) + axis (int): The index for weighted mean + other_axis (int): The other index + + Returns: weighted index for axis, BxC + + """ + mat2line = torch.sum(x, axis=other_axis) + # mat2line = mat2line / mat2line.mean() * 10 + if softmax: + u = torch.softmax(mat2line, axis=2) + else: + u = mat2line / (mat2line.sum(2, keepdim=True) + 1e-6) + size = x.shape[axis] + ind = torch.linspace(0, 1, size).to(x.device) + batch = x.shape[0] + channel = x.shape[1] + index = ind.repeat([batch, channel, 1]) + mean_position = torch.sum(index * u, dim=2) + return mean_position + +def get_expected_points_from_map(hm, softmax=True): + """get_gaussian_map_from_points + B,C,H,W -> B,N,2 float(0, 1) float(0, 1) + softargmax function + + Args: + hm (float): Input images(BxCxHxW) + + Returns: + weighted index for axis, BxCx2. float between 0 and 1. + + """ + # hm = 10*hm + B,C,H,W = hm.shape + y_mean = get_gaussian_mean(hm, 2, 3, softmax=softmax) # B,C + x_mean = get_gaussian_mean(hm, 3, 2, softmax=softmax) # B,C + # return torch.cat((x_mean.unsqueeze(-1), y_mean.unsqueeze(-1)), 2) + return torch.stack([x_mean, y_mean], dim=2) + +# Positional encoding (section 5.1) +# borrow from nerf +class Embedder: + def __init__(self, **kwargs): + self.kwargs = kwargs + self.create_embedding_fn() + + def create_embedding_fn(self): + embed_fns = [] + d = self.kwargs['input_dims'] + out_dim = 0 + if self.kwargs['include_input']: + embed_fns.append(lambda x : x) + out_dim += d + + max_freq = self.kwargs['max_freq_log2'] + N_freqs = self.kwargs['num_freqs'] + + if self.kwargs['log_sampling']: + freq_bands = 2.**torch.linspace(0., max_freq, steps=N_freqs) + else: + freq_bands = torch.linspace(2.**0., 2.**max_freq, steps=N_freqs) + + for freq in freq_bands: + for p_fn in self.kwargs['periodic_fns']: + embed_fns.append(lambda x, p_fn=p_fn, freq=freq : p_fn(x * freq)) + out_dim += d + + self.embed_fns = embed_fns + self.out_dim = out_dim + + def embed(self, inputs): + return torch.cat([fn(inputs) for fn in self.embed_fns], -1) + + +def get_embedder(multires, i=0): + import torch.nn as nn + if i == -1: + return nn.Identity(), 3 + + embed_kwargs = { + 'include_input' : True, + 'input_dims' : 3, + 'max_freq_log2' : multires-1, + 'num_freqs' : multires, + 'log_sampling' : True, + 'periodic_fns' : [torch.sin, torch.cos], + } + + embedder_obj = Embedder(**embed_kwargs) + embed = lambda x, eo=embedder_obj : eo.embed(x) + return embed, embedder_obj.out_dim + +class APOPMeter(): + def __init__(self) -> None: + self.tp = 0 + self.fp = 0 + self.tn = 0 + self.fn = 0 + + def update(self, pred, gt): + """ + Input: + pred, gt: Tensor() + """ + assert pred.shape == gt.shape + self.tp += torch.logical_and(pred == 1, gt == 1).sum().item() + self.fp += torch.logical_and(pred == 1, gt == 0).sum().item() + self.tn += torch.logical_and(pred == 0, gt == 0).sum().item() + self.tn += torch.logical_and(pred == 1, gt == 0).sum().item() + + def update_cm(self, tp, fp, tn, fn): + self.tp += tp + self.fp += fp + self.tn += tn + self.tn += fn + +def inverse_sigmoid(x, eps=1e-5): + x = x.clamp(min=0, max=1) + x1 = x.clamp(min=eps) + x2 = (1 - x).clamp(min=eps) + return torch.log(x1/x2) + +import argparse +from util.slconfig import SLConfig +def get_raw_dict(args): + """ + return the dicf contained in args. + + e.g: + >>> with open(path, 'w') as f: + json.dump(get_raw_dict(args), f, indent=2) + """ + if isinstance(args, argparse.Namespace): + return vars(args) + elif isinstance(args, dict): + return args + elif isinstance(args, SLConfig): + return args._cfg_dict + else: + raise NotImplementedError("Unknown type {}".format(type(args))) + + +def stat_tensors(tensor): + assert tensor.dim() == 1 + tensor_sm = tensor.softmax(0) + entropy = (tensor_sm * torch.log(tensor_sm + 1e-9)).sum() + + return { + 'max': tensor.max(), + 'min': tensor.min(), + 'mean': tensor.mean(), + 'var': tensor.var(), + 'std': tensor.var() ** 0.5, + 'entropy': entropy + } + + +class NiceRepr: + """Inherit from this class and define ``__nice__`` to "nicely" print your + objects. + + Defines ``__str__`` and ``__repr__`` in terms of ``__nice__`` function + Classes that inherit from :class:`NiceRepr` should redefine ``__nice__``. + If the inheriting class has a ``__len__``, method then the default + ``__nice__`` method will return its length. + + Example: + >>> class Foo(NiceRepr): + ... def __nice__(self): + ... return 'info' + >>> foo = Foo() + >>> assert str(foo) == '' + >>> assert repr(foo).startswith('>> class Bar(NiceRepr): + ... pass + >>> bar = Bar() + >>> import pytest + >>> with pytest.warns(None) as record: + >>> assert 'object at' in str(bar) + >>> assert 'object at' in repr(bar) + + Example: + >>> class Baz(NiceRepr): + ... def __len__(self): + ... return 5 + >>> baz = Baz() + >>> assert str(baz) == '' + """ + + def __nice__(self): + """str: a "nice" summary string describing this module""" + if hasattr(self, '__len__'): + # It is a common pattern for objects to use __len__ in __nice__ + # As a convenience we define a default __nice__ for these objects + return str(len(self)) + else: + # In all other cases force the subclass to overload __nice__ + raise NotImplementedError( + f'Define the __nice__ method for {self.__class__!r}') + + def __repr__(self): + """str: the string of the module""" + try: + nice = self.__nice__() + classname = self.__class__.__name__ + return f'<{classname}({nice}) at {hex(id(self))}>' + except NotImplementedError as ex: + warnings.warn(str(ex), category=RuntimeWarning) + return object.__repr__(self) + + def __str__(self): + """str: the string of the module""" + try: + classname = self.__class__.__name__ + nice = self.__nice__() + return f'<{classname}({nice})>' + except NotImplementedError as ex: + warnings.warn(str(ex), category=RuntimeWarning) + return object.__repr__(self) + + + +def ensure_rng(rng=None): + """Coerces input into a random number generator. + + If the input is None, then a global random state is returned. + + If the input is a numeric value, then that is used as a seed to construct a + random state. Otherwise the input is returned as-is. + + Adapted from [1]_. + + Args: + rng (int | numpy.random.RandomState | None): + if None, then defaults to the global rng. Otherwise this can be an + integer or a RandomState class + Returns: + (numpy.random.RandomState) : rng - + a numpy random number generator + + References: + .. [1] https://gitlab.kitware.com/computer-vision/kwarray/blob/master/kwarray/util_random.py#L270 # noqa: E501 + """ + + if rng is None: + rng = np.random.mtrand._rand + elif isinstance(rng, int): + rng = np.random.RandomState(rng) + else: + rng = rng + return rng + +def random_boxes(num=1, scale=1, rng=None): + """Simple version of ``kwimage.Boxes.random`` + + Returns: + Tensor: shape (n, 4) in x1, y1, x2, y2 format. + + References: + https://gitlab.kitware.com/computer-vision/kwimage/blob/master/kwimage/structs/boxes.py#L1390 + + Example: + >>> num = 3 + >>> scale = 512 + >>> rng = 0 + >>> boxes = random_boxes(num, scale, rng) + >>> print(boxes) + tensor([[280.9925, 278.9802, 308.6148, 366.1769], + [216.9113, 330.6978, 224.0446, 456.5878], + [405.3632, 196.3221, 493.3953, 270.7942]]) + """ + rng = ensure_rng(rng) + + tlbr = rng.rand(num, 4).astype(np.float32) + + tl_x = np.minimum(tlbr[:, 0], tlbr[:, 2]) + tl_y = np.minimum(tlbr[:, 1], tlbr[:, 3]) + br_x = np.maximum(tlbr[:, 0], tlbr[:, 2]) + br_y = np.maximum(tlbr[:, 1], tlbr[:, 3]) + + tlbr[:, 0] = tl_x * scale + tlbr[:, 1] = tl_y * scale + tlbr[:, 2] = br_x * scale + tlbr[:, 3] = br_y * scale + + boxes = torch.from_numpy(tlbr) + return boxes + + +class ModelEma(torch.nn.Module): + def __init__(self, model, decay=0.9997, device=None): + super(ModelEma, self).__init__() + # make a copy of the model for accumulating moving average of weights + self.module = deepcopy(model) + self.module.eval() + + self.decay = decay + self.device = device # perform ema on different device from model if set + if self.device is not None: + self.module.to(device=device) + + def _update(self, model, update_fn): + with torch.no_grad(): + for ema_v, model_v in zip(self.module.state_dict().values(), model.state_dict().values()): + if self.device is not None: + model_v = model_v.to(device=self.device) + ema_v.copy_(update_fn(ema_v, model_v)) + + def update(self, model): + self._update(model, update_fn=lambda e, m: self.decay * e + (1. - self.decay) * m) + + def set(self, model): + self._update(model, update_fn=lambda e, m: m) + +class BestMetricSingle(): + def __init__(self, init_res=0.0, better='large') -> None: + self.init_res = init_res + self.best_res = init_res + self.best_ep = -1 + + self.better = better + assert better in ['large', 'small'] + + def isbetter(self, new_res, old_res): + if self.better == 'large': + return new_res > old_res + if self.better == 'small': + return new_res < old_res + + def update(self, new_res, ep): + if self.isbetter(new_res, self.best_res): + self.best_res = new_res + self.best_ep = ep + return True + return False + + def __str__(self) -> str: + return "best_res: {}\t best_ep: {}".format(self.best_res, self.best_ep) + + def __repr__(self) -> str: + return self.__str__() + + def summary(self) -> dict: + return { + 'best_res': self.best_res, + 'best_ep': self.best_ep, + } + + +class BestMetricHolder(): + def __init__(self, init_res=0.0, better='large', use_ema=False) -> None: + self.best_all = BestMetricSingle(init_res, better) + self.use_ema = use_ema + if use_ema: + self.best_ema = BestMetricSingle(init_res, better) + self.best_regular = BestMetricSingle(init_res, better) + + + def update(self, new_res, epoch, is_ema=False): + """ + return if the results is the best. + """ + if not self.use_ema: + return self.best_all.update(new_res, epoch) + else: + if is_ema: + self.best_ema.update(new_res, epoch) + return self.best_all.update(new_res, epoch) + else: + self.best_regular.update(new_res, epoch) + return self.best_all.update(new_res, epoch) + + def summary(self): + if not self.use_ema: + return self.best_all.summary() + + res = {} + res.update({f'all_{k}':v for k,v in self.best_all.summary().items()}) + res.update({f'regular_{k}':v for k,v in self.best_regular.summary().items()}) + res.update({f'ema_{k}':v for k,v in self.best_ema.summary().items()}) + return res + + def __repr__(self) -> str: + return json.dumps(self.summary(), indent=2) + + def __str__(self) -> str: + return self.__repr__() + \ No newline at end of file diff --git a/util/vis_utils.py b/util/vis_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..7fca5785e7bd6b54909edca083e9fa13e6be18a5 --- /dev/null +++ b/util/vis_utils.py @@ -0,0 +1,92 @@ +import cv2 +import numpy as np + +from util.utils import renorm +from util.misc import color_sys + +_color_getter = color_sys(100) + +# plot known and unknown box +def add_box_to_img(img, boxes, colorlist, brands=None): + """[summary] + + Args: + img ([type]): np.array, H,W,3 + boxes ([type]): list of list(4) + colorlist: list of colors. + brands: text. + + Return: + img: np.array. H,W,3. + """ + H, W = img.shape[:2] + for _i, (box, color) in enumerate(zip(boxes, colorlist)): + x, y, w, h = box[0] * W, box[1] * H, box[2] * W, box[3] * H + img = cv2.rectangle(img.copy(), (int(x-w/2), int(y-h/2)), (int(x+w/2), int(y+h/2)), color, 2) + if brands is not None: + brand = brands[_i] + org = (int(x-w/2), int(y+h/2)) + font = cv2.FONT_HERSHEY_SIMPLEX + fontScale = 0.5 + thickness = 1 + img = cv2.putText(img.copy(), str(brand), org, font, + fontScale, color, thickness, cv2.LINE_AA) + return img + +def plot_dual_img(img, boxes, labels, idxs, probs=None): + """[summary] + + Args: + img ([type]): 3,H,W. tensor. + boxes (): tensor(Kx4) or list of tensor(1x4). + labels ([type]): list of ints. + idxs ([type]): list of ints. + probs (optional): listof floats. + + Returns: + img_classcolor: np.array. H,W,3. img with class-wise label. + img_seqcolor: np.array. H,W,3. img with seq-wise label. + """ + + boxes = [i.cpu().tolist() for i in boxes] + img = (renorm(img.cpu()).permute(1,2,0).numpy() * 255).astype(np.uint8) + # plot with class + class_colors = [_color_getter(i) for i in labels] + if probs is not None: + brands = ["{},{:.2f}".format(j,k) for j,k in zip(labels, probs)] + else: + brands = labels + img_classcolor = add_box_to_img(img, boxes, class_colors, brands=brands) + # plot with seq + seq_colors = [_color_getter((i * 11) % 100) for i in idxs] + img_seqcolor = add_box_to_img(img, boxes, seq_colors, brands=idxs) + return img_classcolor, img_seqcolor + + +def plot_raw_img(img, boxes, labels): + """[summary] + + Args: + img ([type]): 3,H,W. tensor. + boxes ([type]): Kx4. tensor + labels ([type]): K. tensor. + + return: + img: np.array. H,W,3. img with bbox annos. + + """ + img = (renorm(img.cpu()).permute(1,2,0).numpy() * 255).astype(np.uint8) + H, W = img.shape[:2] + for box, label in zip(boxes.tolist(), labels.tolist()): + x, y, w, h = box[0] * W, box[1] * H, box[2] * W, box[3] * H + + img = cv2.rectangle(img.copy(), (int(x-w/2), int(y-h/2)), (int(x+w/2), int(y+h/2)), _color_getter(label), 2) + # add text + org = (int(x-w/2), int(y+h/2)) + font = cv2.FONT_HERSHEY_SIMPLEX + fontScale = 1 + thickness = 1 + img = cv2.putText(img.copy(), str(label), org, font, + fontScale, _color_getter(label), thickness, cv2.LINE_AA) + + return img \ No newline at end of file diff --git a/util/visualizer.py b/util/visualizer.py new file mode 100644 index 0000000000000000000000000000000000000000..5eb2a28d6593a5767ec6b78ce4cb482af344879b --- /dev/null +++ b/util/visualizer.py @@ -0,0 +1,130 @@ +# -*- coding: utf-8 -*- +''' +@File : visualizer.py +@Time : 2022/04/05 11:39:33 +@Author : Shilong Liu +@Contact : liusl20@mail.tsinghua.edu.cn; slongliu86@gmail.com +Modified from COCO evaluator +''' + +import os, sys +from textwrap import wrap +import torch +import numpy as np +import cv2 +import datetime + +import matplotlib.pyplot as plt +from matplotlib.collections import PatchCollection +from matplotlib.patches import Polygon +from pycocotools import mask as maskUtils +from matplotlib import transforms + +def renorm(img: torch.FloatTensor, mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) \ + -> torch.FloatTensor: + # img: tensor(3,H,W) or tensor(B,3,H,W) + # return: same as img + assert img.dim() == 3 or img.dim() == 4, "img.dim() should be 3 or 4 but %d" % img.dim() + if img.dim() == 3: + assert img.size(0) == 3, 'img.size(0) shoule be 3 but "%d". (%s)' % (img.size(0), str(img.size())) + img_perm = img.permute(1,2,0) + mean = torch.Tensor(mean) + std = torch.Tensor(std) + img_res = img_perm * std + mean + return img_res.permute(2,0,1) + else: # img.dim() == 4 + assert img.size(1) == 3, 'img.size(1) shoule be 3 but "%d". (%s)' % (img.size(1), str(img.size())) + img_perm = img.permute(0,2,3,1) + mean = torch.Tensor(mean) + std = torch.Tensor(std) + img_res = img_perm * std + mean + return img_res.permute(0,3,1,2) + +class ColorMap(): + def __init__(self, basergb=[255,255,0]): + self.basergb = np.array(basergb) + def __call__(self, attnmap): + # attnmap: h, w. np.uint8. + # return: h, w, 4. np.uint8. + assert attnmap.dtype == np.uint8 + h, w = attnmap.shape + res = self.basergb.copy() + res = res[None][None].repeat(h, 0).repeat(w, 1) # h, w, 3 + attn1 = attnmap.copy()[..., None] # h, w, 1 + res = np.concatenate((res, attn1), axis=-1).astype(np.uint8) + return res + + +class COCOVisualizer(): + def __init__(self) -> None: + pass + + def visualize(self, img, tgt, caption=None, dpi=120, savedir=None, show_in_console=True): + """ + img: tensor(3, H, W) + tgt: make sure they are all on cpu. + must have items: 'image_id', 'boxes', 'size' + """ + plt.figure(dpi=dpi) + plt.rcParams['font.size'] = '5' + ax = plt.gca() + img = renorm(img).permute(1, 2, 0) + ax.imshow(img) + + self.addtgt(tgt) + if show_in_console: + plt.show() + + if savedir is not None: + if caption is None: + savename = '{}/{}-{}.png'.format(savedir, int(tgt['image_id']), str(datetime.datetime.now()).replace(' ', '-')) + else: + savename = '{}/{}-{}-{}.png'.format(savedir, caption, int(tgt['image_id']), str(datetime.datetime.now()).replace(' ', '-')) + print("savename: {}".format(savename)) + os.makedirs(os.path.dirname(savename), exist_ok=True) + plt.savefig(savename) + plt.close() + + def addtgt(self, tgt): + """ + - tgt: dict. args: + - boxes: num_boxes, 4. xywh, [0,1]. + - box_label: num_boxes. + """ + assert 'boxes' in tgt + ax = plt.gca() + H, W = tgt['size'].tolist() + numbox = tgt['boxes'].shape[0] + + color = [] + polygons = [] + boxes = [] + for box in tgt['boxes'].cpu(): + unnormbbox = box * torch.Tensor([W, H, W, H]) + unnormbbox[:2] -= unnormbbox[2:] / 2 + [bbox_x, bbox_y, bbox_w, bbox_h] = unnormbbox.tolist() + boxes.append([bbox_x, bbox_y, bbox_w, bbox_h]) + poly = [[bbox_x, bbox_y], [bbox_x, bbox_y+bbox_h], [bbox_x+bbox_w, bbox_y+bbox_h], [bbox_x+bbox_w, bbox_y]] + np_poly = np.array(poly).reshape((4,2)) + polygons.append(Polygon(np_poly)) + c = (np.random.random((1, 3))*0.6+0.4).tolist()[0] + color.append(c) + + p = PatchCollection(polygons, facecolor=color, linewidths=0, alpha=0.1) + ax.add_collection(p) + p = PatchCollection(polygons, facecolor='none', edgecolors=color, linewidths=2) + ax.add_collection(p) + + + if 'box_label' in tgt: + assert len(tgt['box_label']) == numbox, f"{len(tgt['box_label'])} = {numbox}, " + for idx, bl in enumerate(tgt['box_label']): + _string = str(bl) + bbox_x, bbox_y, bbox_w, bbox_h = boxes[idx] + # ax.text(bbox_x, bbox_y, _string, color='black', bbox={'facecolor': 'yellow', 'alpha': 1.0, 'pad': 1}) + ax.text(bbox_x, bbox_y, _string, color='black', bbox={'facecolor': color[idx], 'alpha': 0.6, 'pad': 1}) + + if 'caption' in tgt: + ax.set_title(tgt['caption'], wrap=True) + + diff --git a/women.jpg b/women.jpg new file mode 100644 index 0000000000000000000000000000000000000000..51b780e07a3bb582e2072f9905cea5c196c1a6de Binary files /dev/null and b/women.jpg differ