anonymize_file_clear.py

import argparse
import json
import re
import uuid
from pathlib import Path
import gensim
from concrete.ml.common.serialization.loaders import load
from transformers import AutoTokenizer, AutoModel
from utils_demo import get_batch_text_representation

def load_models():
    base_dir = Path(__file__).parent / "models"

    # Load tokenizer and model
    tokenizer = AutoTokenizer.from_pretrained("obi/deid_roberta_i2b2")
    embeddings_model = AutoModel.from_pretrained("obi/deid_roberta_i2b2")

    with open(base_dir / "cml_logreg.model", "r") as model_file:
        fhe_ner_detection = load(file=model_file)
    return embeddings_model, tokenizer, fhe_ner_detection

def anonymize_text(text, embeddings_model, tokenizer, fhe_ner_detection):
    token_pattern = r"(\b[\w\.\/\-@]+\b|[\s,.!?;:'\"-]+)"
    tokens = re.findall(token_pattern, text)
    uuid_map = {}
    processed_tokens = []

    for token in tokens:
        if token.strip() and re.match(r"\w+", token):  # If the token is a word
            x = get_batch_text_representation([token], embeddings_model, tokenizer)
            prediction_proba = fhe_ner_detection.predict_proba(x)
            probability = prediction_proba[0][1]
            prediction = probability >= 0.5
            if prediction:
                if token not in uuid_map:
                    uuid_map[token] = str(uuid.uuid4())[:8]
                processed_tokens.append(uuid_map[token])
            else:
                processed_tokens.append(token)
        else:
            processed_tokens.append(token)  # Preserve punctuation and spaces as is

    anonymized_text = ''.join(processed_tokens)
    return anonymized_text, uuid_map

def main():
    parser = argparse.ArgumentParser(description="Anonymize named entities in a text file and save the mapping to a JSON file.")
    parser.add_argument("file_path", type=str, help="The path to the file to be processed.")
    args = parser.parse_args()

    embeddings_model, tokenizer, fhe_ner_detection = load_models()

    # Read the input file
    with open(args.file_path, 'r', encoding='utf-8') as file:
        text = file.read()

    # Save the original text to its specified file
    original_file_path = Path(__file__).parent / "files" / "original_document.txt"
    with open(original_file_path, 'w', encoding='utf-8') as original_file:
        original_file.write(text)
    
    # Anonymize the text
    anonymized_text, uuid_map = anonymize_text(text, embeddings_model, tokenizer, fhe_ner_detection)

    # Save the anonymized text to its specified file
    anonymized_file_path = Path(__file__).parent / "files" / "anonymized_document.txt"
    with open(anonymized_file_path, 'w', encoding='utf-8') as anonymized_file:
        anonymized_file.write(anonymized_text)

    # Save the UUID mapping to a JSON file
    mapping_path = Path(args.file_path).stem + "_uuid_mapping.json"
    with open(mapping_path, 'w', encoding='utf-8') as file:
        json.dump(uuid_map, file, indent=4, sort_keys=True)

    print(f"Original text saved to {original_file_path}")
    print(f"Anonymized text saved to {anonymized_file_path}")
    print(f"UUID mapping saved to {mapping_path}")

if __name__ == "__main__":
    main()