Split report into sentences

utils_casemaker.py

@@ -34,11 +34,20 @@ def clean(s: str) -> str:
     return s
 
 
-def split_paragraphs(text: str) -> List[str]:
+def split_into_sentences(text: str):
+    # Split paragraphs
     paragraphs = text.split("\n\n")
     paragraphs = list(map(clean, paragraphs))
     paragraphs = list(filter(lambda s: len(s.split()) > 10, paragraphs))
-    return paragraphs
+    
+    # Split into sentences
+    sentences = [sent_tokenize(p) for p in paragraphs]
+    sentences = [
+        sent
+        for lst in sentences
+        for sent in lst if isinstance(sent, str)
+    ]
+    return sentences
 
 
 def format_casemaker_data(
@@ -137,8 +146,8 @@ class CaseMaker:
         """
         report_string_by_organ = dict[str, str]()
 
-        # Split the report into a list of paragraphs
-        paragraphs = split_paragraphs(report)
+        # Split the report into a list of sentences
+        paragraphs = split_into_sentences(report)
         # Collect a list of paragraphs related to each organ
         for p in paragraphs:
             # Figure out which organ is being referenced