Upload folder using huggingface_hub

.dockerignore

@@ -0,0 +1,3 @@
+*venv*
+log
+log_run

.gitignore

@@ -0,0 +1,21 @@
+venv/
+data/
+vncore/
+global/
+cls/
+check.json
+test3.py
+test2.py
+time_test.py
+vncorenlp
+*venv*
+__pycache__/
+req_daily
+log*
+sample
+model
+*.json
+*test*
+docker_venv
+core
+models

Dockerfile

@@ -0,0 +1,46 @@
+FROM python:3.7
+
+WORKDIR /app
+
+
+RUN apt-get update && apt-get install build-essential cmake git -y
+
+#----------------JRE (for vncorenlp)--------------
+RUN apt-get update && \
+    DEBIAN_FRONTEND=noninteractive \
+    apt-get -y install default-jre-headless && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN apt-get install unzip
+RUN pip install gdown
+RUN gdown --id 1MTAPYy9AcYtfiJ6m_pz6MPeA6li8pYp7
+RUN unzip vncorenlp.zip -d /app/
+
+
+# COPY ./model /app/model
+
+RUN mkdir -p /app/model
+RUN mkdir -p /app/log
+RUN mkdir -p /app/log_run
+
+COPY reqs_cpu.txt  /app/
+RUN pip install -r reqs_cpu.txt
+
+COPY ./load_model.py /app/
+RUN python load_model.py
+
+COPY ./config /app/config
+COPY ./entity /app/entity
+COPY ./function /app/function
+COPY ./main_cache.py /app/
+COPY ./service_cache.py /app/
+COPY ./summary.py /app/
+COPY ./merge_topic.py /app/
+COPY ./consumer_clustering.py /app/
+COPY ./consumer_merge_clustering.py /app/
+COPY ./run_multi_process.py /app/
+
+RUN rm -r ~/.cache/pip/*
+
+CMD ["python", "run_multi_process.py"]

Dockerfile_gpu

@@ -0,0 +1,33 @@
+FROM python:3.7
+
+WORKDIR /app
+
+
+RUN apt-get update && apt-get install build-essential cmake git -y
+
+#----------------JRE (for vncorenlp)--------------
+RUN apt-get update && \
+    DEBIAN_FRONTEND=noninteractive \
+    apt-get -y install default-jre-headless && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+COPY ./model /app/model
+
+RUN pip install torch==1.9.0+cu111 torchvision==0.10.0+cu111 torchaudio==0.9.0 -f https://download.pytorch.org/whl/torch_stable.html
+RUN mkdir log
+RUN mkdir log_run
+
+COPY req.txt /app/
+RUN pip install -r req.txt
+
+COPY ./entity /app/entity
+COPY ./function /app/function
+COPY ./vncorenlp /app/vncorenlp
+COPY ./main_cache.py /app/
+COPY ./service_cache.py /app/
+COPY ./summary.py /app/
+
+RUN rm -r ~/.cache/pip/*
+
+CMD ["python", "main_cache.py"]

Dockerfile_gpu_Thien

@@ -0,0 +1,34 @@
+FROM python:3.7.17-slim-bullseye
+
+WORKDIR /app
+
+
+RUN apt-get update && apt-get install build-essential cmake git -y
+
+#----------------JRE (for vncorenlp)--------------
+RUN apt-get update && \
+    DEBIAN_FRONTEND=noninteractive \
+    apt-get -y install default-jre-headless && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+COPY ./model /app/model
+COPY ./tensorRT /app/tensorRT
+
+RUN pip install torch==1.9.0+cu111 torchvision==0.10.0+cu111 torchaudio==0.9.0 -f https://download.pytorch.org/whl/torch_stable.html
+RUN mkdir log
+RUN mkdir log_run
+
+COPY req.txt /app/
+RUN pip install -r req.txt
+
+COPY ./entity /app/entity
+COPY ./function /app/function
+COPY ./vncorenlp /app/vncorenlp
+COPY ./main_cache_Thien.py /app/
+COPY ./service_cache_Thien.py /app/
+COPY ./summary.py /app/
+
+RUN rm -r ~/.cache/pip/*
+
+CMD ["python", "main_cache_Thien.py"]

Jenkinsfile

@@ -0,0 +1,22 @@
+node {
+    checkout scm
+
+    env.DOCKER_API_VERSION="1.23"
+    registry_host = env.registry_host
+    appName = "clusteringcpu"
+
+    sh "git rev-parse --short HEAD > commit-id"
+    tag = readFile('commit-id').replace("\n", "").replace("\r", "")
+
+    stage("build"){
+        sh "docker build --tag ${registry_host}/${appName}:${tag} --file Dockerfile ."
+    }
+    stage("push"){
+        sh "docker push ${registry_host}/${appName}:${tag}"
+    }
+    stage("deploy"){
+        sh "sed -i s/{{tag}}/${tag}/g deployment.yaml \
+        && sed -i 's|{{registry_host}}|${registry_host}|g' deployment.yaml \
+        && kubectl ${env.token_kube} apply -f deployment.yaml"
+    }
+}

README.md

@@ -1,12 +1,55 @@
----
-title: Topic Clustering Global Dashboard
-emoji: 🏆
-colorFrom: gray
-colorTo: red
-sdk: gradio
-sdk_version: 4.43.0
-app_file: app.py
-pinned: false
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+---
+title: topic-clustering-global-dashboard
+app_file: service_periodic.py
+sdk: gradio
+sdk_version: 4.43.0
+---
+## Build docker
+```
+docker build -t clustering-dashboard .
+docker run -d --restart=always --name clustering-dashboard clustering-dashboard
+```
+
+## Deploy TRT
+```
+scp -r docker/api_trt/Dockerfile  ./
+docker build -t api-clustering .
+docker run --gpus 1  --dns 8.8.8.8 -it  -p 8633:8633 -v /home/dp04/topic-clustering-global-dashboard:/app -t --name api-clustering api-clustering
+```
+
+### Topic Clustering
+1. API clustering port 8633: dùng cho phân cụm NEWS trong các loại báo cáo ( báo cáo nhanh, báo cáo tổng quan, báo cáo template,...)
+- Command: 
+    ```
+    docker run --gpus all -it --rm -v /home/vietle/topic-clustering:/home/vietle/topic-clustering --name topic_clustering_trt_sb  -p 8636:8633 topic-clustering-trt
+    cd /home/vietle/topic-clustering && CUDA_VISIBLE_DEVICES=<cuda_device_id>  python main_cache.py
+    ```
+- screen: 52097.clustering.trt
+2. API clustering social: Phân cụm tin tức social
+- Command: cd /home/vietle/topic-clustering/ && source docker_venv/bin/activate && uvicorn service_social:app --host 0.0.0.0 --port 8635
+- Screen: 37529.clustering.social
+3. Phân cụm cho global dashboard:
+- API: 
+    ```
+    CMD: systemctl status bzo_clustering_api_cpu.service (port 8634)
+    Screen: 21445.clustering.cpu
+    ```
+- Consumer: 
+    + Phân cụm daily: 
+    ```
+    Command: cd /home/vietle/topic-clustering/ && source docker_venv/bin/activate && OMP_NUM_THREADS=16 CUDA_VISIBLE_DEVICES=-1 python consumer_clustering.py
+    Screen: 16024.bzo.clustering_daily
+    ```
+    + Phân cụm weekly/monthly:
+    ```
+    Command: cd /home/vietle/topic-clustering/ && source docker_venv/bin/activate &&  OMP_NUM_THREADS=16 CUDA_VISIBLE_DEVICES='-1' python consumer_merge_clustering.py
+    Screen: 60767.bzo.clustering_monthly
+    ```
+
+4. Phân cụm điểm tin:
+- Command: 
+    ```
+    docker run --gpus all -it --rm -v /home/vietle/topic-clustering:/home/vietle/topic-clustering --name topic_clustering_trt_sb  -p 8636:8633 topic-clustering-trt
+    cd /home/vietle/topic-clustering/ && python main_cache_Thien.py
+    ```
+       

config/__init__.py

@@ -0,0 +1 @@
+from .config import get_config, parse_connection_string

config/cfg.yaml

@@ -0,0 +1,173 @@
+AppSettings: 
+ConnectionStrings:
+  facebook_info: Server=10.9.3.141,1433; Database=facebook_info; User Id='facebook_info_read'; Password='dh2uxJny'; Pooling=True; Connect Timeout=45; MultipleActiveResultSets=True; Encrypt=False
+GraylogConfig: 
+ConfigManager:
+  ChannelConfigs: 
+  MemcachedConfigs: 
+  QueueConfigs:
+    queue_topic_clustering:
+      Arguments: {}
+      AutoAck: false
+      AutoDelete: false
+      AutomaticRecoveryEnabled: true
+      ConsumerMaxRetry: 0
+      Durable: true
+      ErrorCounter: 100
+      ErrorCounterTotalSeconds: 60
+      Exchange: ''
+      ExchangeArguments: 
+      ExchangeAutoDelete: false
+      ExchangeDurable: true
+      ExchangeType: 
+      Exclusive: false
+      FailedQueue: 
+      HostName: 10.9.3.251
+      HostNames:
+      - 10.9.3.251
+      Servers:
+      - Host: 10.9.3.251
+        Port: 15672
+      Key: queue1
+      MaxWorkpool: 1
+      NetworkRecoveryInterval: 120
+      Password: 1
+      PrefetchCount: 200
+      Queue: topic-clustering
+      QueueBindArguments: 
+      QueueBindRoutingKey: 
+      RequestedHeartbeat: 120
+      TopologyRecoveryEnabled: false
+      UserName: long.nguyen
+      VirtualHost: posts-broadcast
+      MessageDeliveryMode: 
+      IsProducer: true
+      IsConsumer: true
+    queue_merge_clustering:
+      Arguments: {}
+      AutoAck: false
+      AutoDelete: false
+      AutomaticRecoveryEnabled: true
+      ConsumerMaxRetry: 0
+      Durable: true
+      ErrorCounter: 100
+      ErrorCounterTotalSeconds: 60
+      Exchange: ''
+      ExchangeArguments: 
+      ExchangeAutoDelete: false
+      ExchangeDurable: true
+      ExchangeType: 
+      Exclusive: false
+      FailedQueue: 
+      HostName: 10.9.3.251
+      HostNames:
+      - 10.9.3.251
+      Servers:
+      - Host: 10.9.3.251
+        Port: 15672
+      Key: queue2
+      MaxWorkpool: 1
+      NetworkRecoveryInterval: 120
+      Password: 1
+      PrefetchCount: 200
+      Queue: merge-clustering
+      QueueBindArguments: 
+      QueueBindRoutingKey: 
+      RequestedHeartbeat: 120
+      TopologyRecoveryEnabled: false
+      UserName: long.nguyen
+      VirtualHost: posts-broadcast
+      MessageDeliveryMode: 
+      IsProducer: true
+      IsConsumer: true
+    queue_topic_clustering_mnews:
+      Arguments: {}
+      AutoAck: false
+      AutoDelete: false
+      AutomaticRecoveryEnabled: true
+      ConsumerMaxRetry: 0
+      Durable: true
+      ErrorCounter: 100
+      ErrorCounterTotalSeconds: 60
+      Exchange: ''
+      ExchangeArguments: 
+      ExchangeAutoDelete: false
+      ExchangeDurable: true
+      ExchangeType: 
+      Exclusive: false
+      FailedQueue: 
+      HostName: 10.9.3.251
+      HostNames:
+      - 10.9.3.251
+      Servers:
+      - Host: 10.9.3.251
+        Port: 15672
+      Key: queue1
+      MaxWorkpool: 1
+      NetworkRecoveryInterval: 120
+      Password: 1
+      PrefetchCount: 200
+      Queue: topic-clustering-mnews
+      QueueBindArguments: 
+      QueueBindRoutingKey: 
+      RequestedHeartbeat: 120
+      TopologyRecoveryEnabled: false
+      UserName: long.nguyen
+      VirtualHost: posts-broadcast
+      MessageDeliveryMode: 
+      IsProducer: true
+      IsConsumer: true
+    queue_merge_clustering_newscms:
+      Arguments: {}
+      AutoAck: false
+      AutoDelete: false
+      AutomaticRecoveryEnabled: true
+      ConsumerMaxRetry: 0
+      Durable: true
+      ErrorCounter: 100
+      ErrorCounterTotalSeconds: 60
+      Exchange: ''
+      ExchangeArguments: 
+      ExchangeAutoDelete: false
+      ExchangeDurable: true
+      ExchangeType: fanout
+      Exclusive: false
+      FailedQueue: 
+      HostName: 10.9.3.251
+      HostNames:
+      - 10.9.3.251
+      Servers:
+      - Host: 10.9.3.251
+        Port: 5672
+      Key: queue_merge_clustering_newscms
+      MaxWorkpool: 1
+      NetworkRecoveryInterval: 120
+      Password: 1
+      PrefetchCount: 200
+      Queue: newscms-merge-clustering
+      QueueBindArguments: 
+      QueueBindRoutingKey: 
+      RequestedHeartbeat: 120
+      TopologyRecoveryEnabled: false
+      UserName: long.nguyen
+      VirtualHost: news-cms
+      MessageDeliveryMode: 
+      IsProducer: true
+      IsConsumer: true
+  AppSettings: 
+  ApiConnects:
+    api_save_clustering:
+      BaseUrl: https://staging.pontusinc.com/api/category_management/v1/topic-clustering-dashboard/insert
+      IsMonitor: true
+      MonitorUrl: https://staging.pontusinc.com/api/category_management/v1/topic-clustering-dashboard/insert
+    api_sbert:
+      BaseUrl: http://10.9.3.240:6789/sbert/encode_list
+      IsMonitor: true
+      MonitorUrl: http://10.9.3.240:6789/sbert/encode_list
+
+    api_save_cluster_newscms:
+      BaseUrl: https://staging.pontusinc.com/api/news_cms/News/update_cluster_result
+      IsMonitor: true
+      MonitorUrl: https://staging.pontusinc.com/api/news_cms/News/update_cluster_result
+  GraylogConfig: 
+key_test: test

config/config.py

@@ -0,0 +1,57 @@
+import requests
+import json
+import os
+from yaml import load
+try:
+    from yaml import CLoader as Loader, CDumper as Dumper
+except ImportError:
+    from yaml import Loader, Dumper
+
+
+URL_CFG = "http://icomm-api-configserver/api/configserver/v1/configuration.yaml"
+# http://10.9.2.151:31244/api/configserver/v1/configuration.yaml
+# cấu hình file hosts ở thư mục C:\Windows\System32\drivers\etc như sau:
+# 123.31.42.17 icomm-api-configserver
+
+AccessToken = "wbecrEfJk8F36y0WojqBQaqT28d6NaBnCLBgkoO2sCg3aNhYACkSxMNvWwlsAj5k"
+Environment = "Production"
+path_save_cfg = "config/cfg.yaml"
+
+
+def get_config():
+    cfg = None
+    try:
+        payload = json.dumps({
+            "AccessToken": AccessToken,
+            "Environment": Environment
+        })
+        headers = {
+            'accept': 'text/plain',
+            'Content-Type': 'application/json-patch+json'
+        }
+
+        response = requests.request("POST", URL_CFG, headers=headers, data=payload)
+        # if response.status_code == 200:
+        #     with open(path_save_cfg, "w+") as f:
+        #         f.write(response.text)
+    except Exception as ve:
+        print(ve)
+    if os.path.exists(path_save_cfg):
+        with open(path_save_cfg) as f:
+            cfg = load(f, Loader)
+    return cfg
+
+
+def parse_connection_string(str_cnn):
+    res = dict()
+    split_dt = str_cnn.split(";")
+    for c_sp in split_dt:
+        k, v = c_sp.split("=")
+        res[k.strip()] = v.replace("'", "").replace('"', '')
+    return res
+
+
+if __name__ == '__main__':
+    cf = get_config()
+    print(cf)
+    print(parse_connection_string(cf["ConfigManager"]["ConnectionStrings"]["facebook_info"]["Value"]))

config/config.yml

@@ -0,0 +1,38 @@
+queue_topic_clustering:
+  host:
+    10.9.3.251
+  virtual_host:
+    posts-broadcast
+  queue_name:
+    topic-clustering
+  usr_name:
+    long.nguyen
+  password:
+    1
+
+queue_topic_clustering_mnews:
+  host:
+    10.9.3.251
+  virtual_host:
+    posts-broadcast
+  queue_name:
+    topic-clustering-mnews
+  usr_name:
+    long.nguyen
+  password:
+    1
+
+queue_merge_clustering:
+  host:
+    10.9.3.251
+  virtual_host:
+    posts-broadcast
+  queue_name:
+    merge-clustering
+  usr_name:
+    long.nguyen
+  password:
+    1
+
+api_save_clustering:
+  https://staging.pontusinc.com/api/category_management/v1/topic-clustering-dashboard/insert

consumer_clustering.py

@@ -0,0 +1,181 @@
+import pika
+import os
+# os.environ['CUDA_VISIBLE_DEVICES'] = '-1'
+
+import json
+import time
+# from get_config import config_params
+from config import get_config
+from function import topic_clustering_not_summary as tc
+from function import topic_clustering_social
+import requests
+
+config_params = get_config()
+ConfigManager = config_params['ConfigManager']
+
+last_time_check = time.time()
+def update_result(result, type='daily', meta = {}):
+    benchmark_children_id = -1
+    benchmark_id = -1
+    source_tagids = []
+    for id_cluster in result:
+        for doc in result[id_cluster][:1]:
+            source_tagids = doc.get('source_tagids',[])
+            for key in doc:
+                if "benchmark_child" in key:
+                    benchmark_children_id = int(key.lstrip('benchmark_child_'))
+                if "benchmark" in key and 'child' not in key:  
+                    benchmark_id = int(key.lstrip('benchmark_'))
+        break
+    
+    if not source_tagids:
+        source_tagids = []
+
+    if len(source_tagids) > 0:
+        benchmark_id = 0
+        benchmark_children_id = 0
+
+    output = {
+        "benchmark_id": benchmark_id,
+        "benchmark_children_id": benchmark_children_id,
+        "source_tagids": source_tagids,
+        "country_code": meta.get('country_code',''),
+        "type": type,
+        "data": json.dumps(result)
+    }
+    # with open('test_result.json','w') as f:
+    #     json.dump(output, f, ensure_ascii=False)
+
+    # url = config_params['api_save_clustering']
+    url = ConfigManager['ApiConnects']['api_save_clustering']['BaseUrl']
+    # with open("/home/vietle/topic-clustering/config/save.json", 'w') as f:
+    #     json.dump(output, f,ensure_ascii=False)
+    res = requests.post(url, json = output)
+    print(res.text)
+    print('Update result !!!!!!!!!')
+
+def callback_func(ch, method, properties, body):
+    print("receive done: ")
+    
+    
+    
+    starttime = time.time()
+    body = json.loads(body.decode("utf-8"))
+    
+    # with open('input_daily.json','w') as f:
+    #     json.dump(body, f, ensure_ascii=False)
+
+    docs = body['docs']
+    # threshold = body['threshold']
+    threshold = 0.25
+    top_cluster = body['top_cluster']
+    top_sentence = body['top_sentence']
+    topn_summary = body['topn_summary']
+    hash_str = body['hash_str']
+    st_time = body['st_time']
+    meta = body.get('meta',{})
+    country_code = meta.get("country_code", "")
+    delete_message = False if country_code in ["ICOMM-RND","SOCIAL"] else True
+
+    print("country_code: ", country_code, "meta: ", meta)
+
+    is_cache = False
+    try:
+        with open("log_run/log.txt") as f:
+            data_dict = json.load(f)
+    except Exception as ve:
+        print(ve)
+        data_dict = {}
+
+    try:
+        if hash_str in data_dict:
+            path_res = data_dict[hash_str]["response_path"]
+            with open(path_res) as ff:
+                results = json.load(ff)
+            print("time analysis (cache): ", time.time() - st_time)
+            update_result(results,meta=meta)
+            is_cache = True
+    except Exception as vee:
+        print(vee)
+
+    if not is_cache:
+        if country_code in ["SOCIAL"]:
+            results = topic_clustering_social.topic_clustering(docs, threshold, top_cluster=top_cluster, top_sentence=top_sentence,
+                                    topn_summary=topn_summary, delete_message=delete_message)
+        else:
+            results = tc.topic_clustering(docs, threshold, top_cluster=top_cluster, top_sentence=top_sentence,
+                                        topn_summary=topn_summary, delete_message=delete_message)
+        update_result(results, meta=meta)
+        
+        path_res = "log/result_{0}.txt".format(hash_str)
+        with open(path_res, "w+") as ff:
+            ff.write(json.dumps(results))
+
+        data_dict[hash_str] = {"time": st_time, "response_path": path_res}
+
+        lst_rm = []
+        global last_time_check
+        if time.time() - last_time_check > 3600:
+            print("check log to del .....")
+            last_time_check = time.time()
+            for dt in data_dict:
+                if time.time() - data_dict[dt]["time"] > 30 * 24 * 3600:
+                    lst_rm.append(dt)
+            for dt in lst_rm:
+                del data_dict[dt]
+            with open("log_run/log.txt", "w+") as ff:
+                ff.write(json.dumps(data_dict))
+        print("time analysis: ", time.time() - starttime)
+    ch.basic_ack(delivery_tag=method.delivery_tag)
+
+
+def test():
+    with open('req_daily/aus.json') as f:
+        body = json.load(f)
+
+    docs = body['response']['docs']
+    # threshold = body['threshold']
+    threshold = 0.25
+    top_cluster = body['top_cluster']
+    top_sentence = body['top_sentence']
+    topn_summary = body['topn_summary']
+    # hash_str = body['hash_str']
+    # st_time = body['st_time']
+    meta = body['response'].get('meta',{})
+    results = tc.topic_clustering(docs, threshold, top_cluster=top_cluster, top_sentence=top_sentence,
+                                    topn_summary=topn_summary, delete_message=True)
+    print(results)
+    # update_result(results, meta=meta)
+    # print(123)
+if __name__ == '__main__':
+    # test()
+    params = ConfigManager['QueueConfigs']['queue_topic_clustering']
+    usr_name = params["UserName"]
+    password = str(params["Password"])
+    host = params["HostName"]
+    virtual_host = params["VirtualHost"]
+    queue_name = params["Queue"]
+
+    
+
+    # params = config_params['queue_topic_clustering']
+    # usr_name = params["usr_name"]
+    # password = str(params["password"])
+    # host = params["host"]
+    # virtual_host = params["virtual_host"]
+    # queue_name = params["queue_name"]
+
+    while True:
+        try:
+            credentials = pika.PlainCredentials(usr_name, password)
+            connection = pika.BlockingConnection(
+                pika.ConnectionParameters(host=host, virtual_host=virtual_host, credentials=credentials, heartbeat=3600, blocked_connection_timeout=3600))
+            channel = connection.channel()
+            channel.queue_declare(queue=queue_name, durable=True, arguments={"x-max-priority": 10})
+            print(" * wait message")
+            channel.basic_qos(prefetch_count=1)
+            channel.basic_consume(queue=queue_name, on_message_callback=callback_func)
+            channel.start_consuming()
+        except Exception as ex:
+            print(f'[ERROR] ', ex)
+            # raise ex

consumer_clustering_mnews.py

@@ -0,0 +1,137 @@
+import pika
+import os
+# os.environ['CUDA_VISIBLE_DEVICES'] = '-1'
+
+import json
+import time
+# from get_config import config_params
+from config import get_config
+from function import topic_clustering_mnews as tc
+import requests
+
+config_params = get_config()
+ConfigManager = config_params['ConfigManager']
+
+last_time_check = time.time()
+def update_result(result, type='daily', meta = {}):
+    command_id = meta["command_id"]
+
+    output = {
+        "id": command_id,
+        "status": 2,
+        "json_result": json.dumps(result)
+    }
+ 
+    url = "https://staging.pontusinc.com/api/news_management/v1/quick_search/update-command-result"
+    # with open("/home/vietle/topic-clustering/config/save_.json", 'w') as f:
+    #     json.dump(output, f,ensure_ascii=False)
+    res = requests.post(url, json = output)
+    print(res.text)
+    print('Update result !!!!!!!!!')
+
+def callback_func(ch, method, properties, body):
+    print("receive done: ")
+    starttime = time.time()
+    body = json.loads(body.decode("utf-8"))
+    
+    # with open('input_daily.json','w') as f:
+    #     json.dump(body, f, ensure_ascii=False)
+
+    docs = body['docs']
+    # threshold = body['threshold']
+    command_id = body.get("command_id",0)
+    print(command_id)
+    threshold = 0.1
+    top_cluster = body['top_cluster']
+    top_sentence = body['top_sentence']
+    topn_summary = body['topn_summary']
+    hash_str = body['hash_str']
+    st_time = body['st_time']
+    meta = body.get('meta',{})
+    meta["command_id"] = command_id
+
+    is_cache = False
+    try:
+        with open("log_mnews/log/log.txt") as f:
+            data_dict = json.load(f)
+    except Exception as ve:
+        print(ve)
+        data_dict = {}
+
+    try:
+        if hash_str in data_dict:
+            path_res = data_dict[hash_str]["response_path"]
+            with open(path_res) as ff:
+                results = json.load(ff)
+            print("time analysis (cache): ", time.time() - st_time)
+            update_result(results,meta=meta)
+            is_cache = True
+    except Exception as vee:
+        print(vee)
+
+    if not is_cache:
+        results = tc.topic_clustering(docs, threshold, top_cluster=top_cluster, top_sentence=top_sentence,
+                                    topn_summary=topn_summary)
+        update_result(results, meta=meta)
+        
+        path_res = "log_mnews/result_{0}.txt".format(hash_str)
+        with open(path_res, "w+") as ff:
+            ff.write(json.dumps(results))
+
+        data_dict[hash_str] = {"time": st_time, "response_path": path_res}
+
+        lst_rm = []
+        global last_time_check
+        if time.time() - last_time_check > 3600:
+            print("check log to del .....")
+            last_time_check = time.time()
+            for dt in data_dict:
+                if time.time() - data_dict[dt]["time"] > 30 * 24 * 3600:
+                    lst_rm.append(dt)
+            for dt in lst_rm:
+                del data_dict[dt]
+            with open("log_mnews/log/log.txt", "w+") as ff:
+                ff.write(json.dumps(data_dict))
+        print("time analysis: ", time.time() - starttime)
+    ch.basic_ack(delivery_tag=method.delivery_tag)
+
+
+def test():
+    with open('req_daily/aus.json') as f:
+        body = json.load(f)
+
+    docs = body['response']['docs']
+    # threshold = body['threshold']
+    threshold = 0.25
+    top_cluster = body['top_cluster']
+    top_sentence = body['top_sentence']
+    topn_summary = body['topn_summary']
+    # hash_str = body['hash_str']
+    # st_time = body['st_time']
+    meta = body['response'].get('meta',{})
+    results = tc.topic_clustering(docs, threshold, top_cluster=top_cluster, top_sentence=top_sentence,
+                                    topn_summary=topn_summary)
+    update_result(results, meta=meta)
+    print(123)
+if __name__ == '__main__':
+    params = ConfigManager['QueueConfigs']['queue_topic_clustering_mnews']
+    usr_name = params["UserName"]
+    password = str(params["Password"])
+    host = params["HostName"]
+    virtual_host = params["VirtualHost"]
+    queue_name = params["Queue"]
+
+    while True:
+        try:
+            credentials = pika.PlainCredentials(usr_name, password)
+            connection = pika.BlockingConnection(
+                pika.ConnectionParameters(host=host, virtual_host=virtual_host, credentials=credentials, heartbeat=3600, blocked_connection_timeout=3600))
+            channel = connection.channel()
+            channel.queue_declare(queue=queue_name, durable=True, arguments={"x-max-priority": 10})
+            print(" * wait message")
+            channel.basic_qos(prefetch_count=1)
+            channel.basic_consume(queue=queue_name, on_message_callback=callback_func)
+            channel.start_consuming()
+        except Exception as ex:
+            print(f'[ERROR] ', ex)
+            # raise ex

consumer_hot_topic_ondemand.py

@@ -0,0 +1,209 @@
+from fastapi import FastAPI
+from fastapi.middleware.cors import CORSMiddleware
+from entity import InputHotTopic, ResponseQueue
+from threading import Thread
+from queue import Queue
+import time
+import json, requests
+from service_cache_Thien import get_data_sorl
+import os
+from datetime import datetime
+from email_validator import push_msg_tele
+
+import time
+import json
+import hashlib
+
+
+from pydantic import BaseModel
+
+class InputHotTopic(BaseModel):
+    start_time: str = "2024-09-03 23:00:00"
+    end_time: str = "2024-09-05 23:00:00"
+    query: str = "Giá nhà chung cư trên Hà Nội"
+    keywords: list = ["chung cư, Hà Nội", "Hoà Lạc"]
+    top_cluster: int = 5
+    prompt: str = """Trong 300 từ, hãy tổng hợp thành một đoạn văn một cách đầy đủ, chi tiết, và trung thực về các chủ đề xung quanh biến động giá nhà chung cư Hà Nội từ nội dung dưới đây.
+Nếu không có thông tin gì liên quan đến giá nhà chung cư Hà Nội trong nội dung cung cấp thì trả lời "không có thông tin".  Không đưa quan điểm cá nhân, không lặp lại một phần câu hỏi, loại bỏ phần mở đầu. Không có những câu từ liên kết như: "Sau đây là nội dung tóm tắt", "Nội dung tóm tắt là", "Dưới đây là " ... """
+    check_relevent: str = "Hãy đánh giá nội dung dưới đây có thông tin liên quan đến giá cả nhà chung cư Hà Nội hay không? Chỉ trả lời có hoặc không, không đưa thêm thông tin không liên quan"
+    max_posts: int = 5000
+    
+def get_hash_id(item: InputHotTopic):
+    str_hash = ""
+    if item.id_topic:
+        str_hash += item.id_topic
+        str_hash += item.start_time
+        return hashlib.sha224(str_hash.encode("utf-8")).hexdigest()
+    else:
+        return ""
+    
+class SessionProcess(object):
+
+    def __init__(self):
+        self.session = dict()
+        
+    def hash_session(self, query: InputHotTopic):
+        hash_dict = query.dict()
+        hash_dict['time'] = int(time.time())
+        return hashlib.sha224(json.dumps(hash_dict).encode("utf-8")).hexdigest()
+    
+    def insert_session(self, data_input):
+        print('data_input: ', data_input)
+        # if self.mode == "command_center":
+        #     hash_id = hash_session(data_input)
+        # else:
+        hash_id = self.hash_session(data_input)
+        if hash_id not in self.session:
+            self.session[hash_id] = {"status": 0, "created_time": time.time(), "update_time": time.time(),
+                                     "result": {}, "data": data_input}
+        return hash_id
+
+    def get_info_session(self, hash_id: str):
+        if hash_id in self.session:
+            return self.session[hash_id]
+        return {"status": -2, "result": {}, "meta": {}}
+
+    def update_session(self, hash_id: str, result: dict, status: int):
+        if hash_id in self.session:
+            self.session[hash_id]["status"] = status
+            self.session[hash_id]["result"] = result
+            self.session[hash_id]["update_time"] = time.time()
+            return True
+        return False
+
+    def delete_session(self, hash_id: str):
+        if hash_id in self.session:
+            del self.session[hash_id]
+            return True
+        return False
+
+SESSION = SessionProcess()
+app = FastAPI(title="Hot Topic")
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+NUM_OF_THREAD = 2
+QQ = Queue(maxsize=0) # don't limit queue
+
+def process_wc():
+    print('Run thr')
+    global SESSION, QQ
+    while True:
+        if not QQ.empty():
+            hash_id = QQ.get()
+            SESSION.update_session(hash_id, {}, 0)
+            print("update trạng thái status = 0: đang xử lý")
+            try:
+                ss_info = SESSION.get_info_session(hash_id)
+                status = ss_info["status"]
+                print("trạng thái hiện tại: ", status)
+                if status == 0:
+                    data_input = SESSION.session[hash_id]["data"]
+                    res_doc = get_data_sorl(data_input.query, data_input.keywords, data_input.start_time, data_input.end_time, max_posts = data_input.max_posts)
+                    print('lenght res_doc: ', len(res_doc))
+                    if not res_doc:
+                        SESSION.update_session(hash_id, {}, -1)
+                    else: 
+                        # start_time: str = "2024-03-03 23:00:00"
+                        current_time = datetime.now()
+                        time_now = current_time.strftime("%Y-%m-%d %H:%M:%S")
+                        d = {
+                            "id_topic": "99999",
+                            "start_time": time_now,
+                            "end_time": data_input.end_time,
+                            "threshold": 0.3,
+                            "top_sentence": -1,
+                            "top_cluster": data_input.top_cluster,
+                            "topn_summary": 10,
+                            "type_cluster": "",
+                            "lang_process": "",
+                            "prompt": data_input.prompt,
+                            "topic_name": data_input.check_relevent,
+                            "responseHeader": {},
+                            "benchmark_topics": [],
+                            "response": {"docs": res_doc}
+                        }
+                        
+                        str_hash = ""
+                        str_hash += "99999"
+                        str_hash += time_now
+                        hash_id_path = hashlib.sha224(str_hash.encode("utf-8")).hexdigest()
+                        
+                        st_time = time.time()
+                        try:
+                            response = requests.post('http://10.9.3.241:8636/newsanalysis/topic_clustering', json=d, timeout=5)
+                        except:
+                            print("Timeout done")
+                            
+                        print("push done msg")
+                        res_clus = {}
+                        # flag = False
+                        # count = 0
+                        # while not flag and count < 18000:
+                        #     if os.path.exists("/home/vietle/topic-clustering/log/result_{0}.txt".format(hash_id_path)):
+                        #         path_res = "/home/vietle/topic-clustering/log/result_{0}.txt".format(hash_id_path)
+                        #         with open(path_res, encoding="utf-8") as ff:
+                        #             res_clus = json.load(ff)
+                        #             res_clus["num_articles"] = len(res_doc)
+                        #             message = "Hello"
+                        #             push_msg_tele(data_input.bot_token , data_input.chat_id , message)
+                        #             print('done processing result')
+                        #             flag = True
+                        #     time.sleep(1)
+                        #     count +=1
+                        #     print('sleep: ', count)
+                            
+                            
+                        print("update done msg")
+                        SESSION.update_session(hash_id_path, res_clus, 1)
+            except Exception as ve_:
+                print(ve_)
+                SESSION.update_session(hash_id_path, {}, -1)
+                raise ve_
+        else:
+            time.sleep(2)
+
+
+for _ in range(NUM_OF_THREAD):
+    worker = Thread(target=process_wc, args=())
+    worker.setDaemon(True)
+    worker.start()
+    
+    
+@app.post("/api/v1/send_message")
+def send_requests(item: InputHotTopic):
+    global SESSION
+    hash_id = SESSION.insert_session(item)
+    if SESSION.session[hash_id]["status"] == 0:
+        QQ.put(hash_id)
+
+    return ResponseQueue(statusCode=1, message="Push to queue done !", result={"hash_id": hash_id})
+
+class InputSession(BaseModel):
+    hash_id: str = ""
+    
+class Response(BaseModel):
+    statusCode: int = 200
+    message: str = ""
+    result: dict = {}
+@app.post("/api/mining/qna/result")
+def get_result(item: InputSession):
+    global SESSION
+    res = SESSION.get_info_session(item.hash_id)
+    status = res["status"]
+    res = res["result"]
+    if status == -1:
+        msg = "ERROR"
+    elif status == 0:
+        msg = "processing ..."
+    elif status == 1:
+        msg = "done"
+        # SESSION.delete_session(item.hash_id)
+    else:
+        msg = "nothing"
+    return Response(statusCode=status, message=msg, result=res)

consumer_merge_clustering.py

@@ -0,0 +1,96 @@
+# import os
+# os.environ["CUDA_VISIBLE_DEVICES"] ="-1"
+import pika
+import json
+import time
+import requests
+
+from merge_topic import main
+# from get_config import config_params
+from config import get_config
+
+config_params = get_config()
+ConfigManager = config_params['ConfigManager']
+
+def update_result(result, type='daily', meta = {}):
+    benchmark_children_id = -1
+    benchmark_id = -1
+    source_tagids = []
+    for id_cluster in result:
+        for doc in result[id_cluster][:1]:
+            source_tagids = doc.get('source_tagids',[])
+            for key in doc:
+                if "benchmark_child" in key:
+                    benchmark_children_id = int(key.lstrip('benchmark_child_'))
+                if "benchmark" in key and 'child' not in key:  
+                    benchmark_id = int(key.lstrip('benchmark_'))
+        break
+    
+    if not source_tagids:
+        source_tagids = []
+    if len(source_tagids) > 0:
+        benchmark_id = 0
+        benchmark_children_id = 0
+
+    output = {
+        "benchmark_id": benchmark_id,
+        "benchmark_children_id": benchmark_children_id,
+        "source_tagids": source_tagids,
+        "country_code": meta.get('country_code',''),
+        "type": type,
+        "data": json.dumps(result)
+    }
+
+    # with open('test_result.json','w') as f:
+    #     json.dump(output, f, ensure_ascii=False)
+    
+    # url = config_params['api_save_clustering']
+    url = ConfigManager['ApiConnects']['api_save_clustering']['BaseUrl']
+
+    res = requests.post(url, json = output)
+    print(res.text)
+    print('Update result !!!!!!!!!')
+
+def callback_func(ch, method, properties, body):
+    print("receive done: ")
+    starttime = time.time()
+    body = json.loads(body.decode("utf-8"))
+    
+    req = body
+    type = req['type']
+    meta = req.get('meta', {})
+    res = main(req)
+    update_result(res, type, meta=meta)
+    print('Time process:', time.time() - starttime)
+    ch.basic_ack(delivery_tag=method.delivery_tag)
+
+
+if __name__ == '__main__':
+    params = ConfigManager['QueueConfigs']['queue_merge_clustering']
+    usr_name = params["UserName"]
+    password = str(params["Password"])
+    host = params["HostName"]
+    virtual_host = params["VirtualHost"]
+    queue_name = params["Queue"]
+
+    # params = config_params['queue_merge_clustering']
+    # usr_name = params["usr_name"]
+    # password = str(params["password"])
+    # host = params["host"]
+    # virtual_host = params["virtual_host"]
+    # queue_name = params["queue_name"]
+    
+    while True:
+        try:
+            credentials = pika.PlainCredentials(usr_name, password)
+            connection = pika.BlockingConnection(
+                pika.ConnectionParameters(host=host, virtual_host=virtual_host, credentials=credentials, heartbeat=3600, blocked_connection_timeout=3600))
+            channel = connection.channel()
+            channel.queue_declare(queue=queue_name, durable=True, arguments={"x-max-priority": 10})
+            print(" * wait message")
+            channel.basic_qos(prefetch_count=1)
+            channel.basic_consume(queue=queue_name, on_message_callback=callback_func)
+            channel.start_consuming()
+        except Exception as ex:
+            print(f'[ERROR] ', ex)
+            # raise ex

consumer_merge_clustering_newscms.py

@@ -0,0 +1,109 @@
+# import os
+# os.environ['CUDA_VISIBLE_DEVICES'] = "-1"
+
+import pika
+import json
+import time
+import requests
+
+from merge_topic import main
+# from get_config import config_params
+from config import get_config
+
+config_params = get_config()
+ConfigManager = config_params['ConfigManager']
+URL_SAVE_CLUSTERING_CMS = ConfigManager["ApiConnects"]["api_save_cluster_newscms"]["BaseUrl"]
+
+def update_result(result, id, meta = {}):
+    print(result)
+    print("-----")
+    output = {
+        "id": id,
+        "result":json.dumps(result)
+    }
+    res = requests.post(url=URL_SAVE_CLUSTERING_CMS, json = output)
+    print(res.text)
+    print('Update result !!!!!!!!!')
+
+def callback_func(ch, method, properties, body):
+    print("receive done: ")
+    starttime = time.time()
+    body = json.loads(body.decode("utf-8"))
+    with open("/home/vietle/topic-clustering/input_merge1.json", 'w') as f:
+        json.dump(body,f,ensure_ascii = False)
+    req = body
+    req["type"] = "monthly" 
+    id = req["id"]
+    meta = req.get('meta', {})
+
+    preprocess_reformat = []
+    preprocess = req["preprocess"]
+    for daily_clusters in preprocess:
+        
+        dict_cluster = {}
+        for i,doc in enumerate(daily_clusters["topic"]):
+            reps_post = doc
+            lst_doc = [reps_post]
+            lst_doc.extend(doc.get("list_posts", []))
+            dict_cluster[i] = lst_doc
+        it = {
+            "topic": dict_cluster
+        }
+        preprocess_reformat.append(it)
+    req["preprocess"] = preprocess_reformat
+
+    res = main(req)
+    update_result(res, id, meta=meta)
+    print('Time process:', time.time() - starttime)
+    ch.basic_ack(delivery_tag=method.delivery_tag)
+
+
+def test():
+    with open("/home/vietle/topic-clustering/input_merge1.json", 'r') as f:
+        body = json.load(f)
+
+    req = body
+    req["type"] = "monthly" 
+    id = req["id"]
+    meta = req.get('meta', {})
+
+    preprocess_reformat = []
+    preprocess = req["preprocess"]
+    for daily_clusters in preprocess:
+        
+        dict_cluster = {}
+        for i,topic in enumerate(daily_clusters["topic"]):
+            dict_cluster[i] = topic
+        it = {
+            "topic": dict_cluster
+        }
+        preprocess_reformat.append(it)
+    req["preprocess"] = preprocess_reformat
+    with open("/home/vietle/topic-clustering/input_merge2.json", 'w') as f:
+        json.dump(req,f,ensure_ascii = False)
+    res = main(req)
+
+if __name__ == '__main__':
+    # test()
+    params = ConfigManager['QueueConfigs']['queue_merge_clustering_newscms']
+    usr_name = params["UserName"]
+    password = str(params["Password"])
+    host = params["HostName"]
+    virtual_host = params["VirtualHost"]
+    queue_name = params["Queue"]
+
+    
+    while True:
+        try:
+            credentials = pika.PlainCredentials(usr_name, password)
+            connection = pika.BlockingConnection(
+                pika.ConnectionParameters(host=host, virtual_host=virtual_host, credentials=credentials, heartbeat=3600, blocked_connection_timeout=3600))
+            channel = connection.channel()
+            channel.queue_declare(queue=queue_name, durable=True, arguments={"x-max-priority": 10})
+            print(" * wait message")
+            channel.basic_qos(prefetch_count=1)
+            channel.basic_consume(queue=queue_name, on_message_callback=callback_func)
+            channel.start_consuming()
+        except Exception as ex:
+            print(f'[ERROR] ', ex)
+            raise ex

deployment.yaml

@@ -0,0 +1,28 @@
+apiVersion: apps/v1beta1
+kind: Deployment
+metadata:
+  name: clusteringcpu
+  labels:
+    app: clusteringcpu
+spec:
+  strategy:
+    type: Recreate
+  template:
+    metadata:
+      labels:
+        app: clusteringcpu
+        tier: clusteringcpu
+    spec:
+      containers:
+      - image: {{registry_host}}/clusteringcpu:{{tag}}
+        name: clusteringcpu
+        resources:
+          requests:
+            cpu: 6
+            memory: 60000Mi
+          limits:
+            cpu: 10
+            memory: 100000Mi
+        ports:
+        - containerPort: 
+          name: 

docker/api_trt/Dockerfile

@@ -0,0 +1,22 @@
+FROM nvcr.io/nvidia/tensorrt:22.07-py3
+
+EXPOSE 8633
+WORKDIR /app
+
+RUN echo "nameserver 8.8.8.8" > /etc/resolv.conf && pip install --upgrade pip
+
+RUN echo "nameserver 8.8.8.8" > /etc/resolv.conf && apt-get update && \
+    DEBIAN_FRONTEND=noninteractive \
+    apt-get -y install default-jre-headless && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+COPY req.txt /app/
+RUN echo "nameserver 8.8.8.8" > /etc/resolv.conf && pip install -r req.txt
+RUN echo "nameserver 8.8.8.8" > /etc/resolv.conf &&    pip install --upgrade pip  && pip3 install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cu113
+
+COPY ./main_cache.py /app/
+
+
+RUN rm -r ~/.cache/pip/*
+CMD ["python3", "main_cache.py"]

docker/cpu/Dockerfile

@@ -0,0 +1,46 @@
+FROM python:3.7
+
+WORKDIR /app
+
+
+RUN apt-get update && apt-get install build-essential cmake git -y
+
+#----------------JRE (for vncorenlp)--------------
+RUN apt-get update && \
+    DEBIAN_FRONTEND=noninteractive \
+    apt-get -y install default-jre-headless && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN apt-get install unzip
+RUN pip install gdown
+RUN gdown --id 1MTAPYy9AcYtfiJ6m_pz6MPeA6li8pYp7
+RUN unzip vncorenlp.zip -d /app/
+
+
+# COPY ./model /app/model
+
+RUN mkdir -p /app/model
+RUN mkdir -p /app/log
+RUN mkdir -p /app/log_run
+
+COPY reqs_cpu.txt  /app/
+RUN pip install -r reqs_cpu.txt
+
+COPY ./load_model.py /app/
+RUN python load_model.py
+
+COPY ./config /app/config
+COPY ./entity /app/entity
+COPY ./function /app/function
+COPY ./main_cache.py /app/
+COPY ./service_cache.py /app/
+COPY ./summary.py /app/
+COPY ./merge_topic.py /app/
+COPY ./consumer_clustering.py /app/
+COPY ./consumer_merge_clustering.py /app/
+COPY ./run_multi_process.py /app/
+
+RUN rm -r ~/.cache/pip/*
+
+CMD ["python", "run_multi_process.py"]

docker/gpu/Dockerfile_gpu

@@ -0,0 +1,33 @@
+FROM python:3.7
+
+WORKDIR /app
+
+
+RUN apt-get update && apt-get install build-essential cmake git -y
+
+#----------------JRE (for vncorenlp)--------------
+RUN apt-get update && \
+    DEBIAN_FRONTEND=noninteractive \
+    apt-get -y install default-jre-headless && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+COPY ./model /app/model
+
+RUN pip install torch==1.9.0+cu111 torchvision==0.10.0+cu111 torchaudio==0.9.0 -f https://download.pytorch.org/whl/torch_stable.html
+RUN mkdir log
+RUN mkdir log_run
+
+COPY req.txt /app/
+RUN pip install -r req.txt
+
+COPY ./entity /app/entity
+COPY ./function /app/function
+COPY ./vncorenlp /app/vncorenlp
+COPY ./main_cache.py /app/
+COPY ./service_cache.py /app/
+COPY ./summary.py /app/
+
+RUN rm -r ~/.cache/pip/*
+
+CMD ["python", "main_cache.py"]

docker_build.txt

@@ -0,0 +1,6 @@
+docker build -t topic_clustering .
+docker run -d --restart=always -p8633:8633  --name topic_clustering topic_clustering
+/////docker run -p8633:8633  --name topic_clustering topic_clustering
+
+docker build -t clustering-dashboard .
+docker run --name clustering-dashboard clustering-dashboard

entity/__init__.py

@@ -0,0 +1 @@
+from .types import *

function/SessionProcess.py

@@ -0,0 +1,55 @@
+
+import time
+import json
+import hashlib
+
+
+from pydantic import BaseModel
+
+class InputHotTopic(BaseModel):
+    start_time: str = ""
+    end_time: str = ""
+    query: str = ""
+    keywords: list = []
+    top_cluster: int = 5
+    prompt: str = ""
+    check_relevent: str = ""
+class SessionProcess(object):
+
+    def __init__(self):
+        self.session = dict()
+        
+    def hash_session(self, query: InputHotTopic):
+        hash_dict = query.dict()
+        hash_dict['time'] = int(time.time())
+        return hashlib.sha224(json.dumps(hash_dict).encode("utf-8")).hexdigest()
+    
+    def insert_session(self, data_input):
+        print('data_input: ', data_input)
+        # if self.mode == "command_center":
+        #     hash_id = hash_session(data_input)
+        # else:
+        hash_id = self.hash_session(data_input)
+        if hash_id not in self.session:
+            self.session[hash_id] = {"status": 0, "created_time": time.time(), "update_time": time.time(),
+                                     "result": {}, "data": data_input}
+        return hash_id
+
+    def get_info_session(self, hash_id: str):
+        if hash_id in self.session:
+            return self.session[hash_id]
+        return {"status": -2, "result": {}, "meta": {}}
+
+    def update_session(self, hash_id: str, result: dict, status: int):
+        if hash_id in self.session:
+            self.session[hash_id]["status"] = status
+            self.session[hash_id]["result"] = result
+            self.session[hash_id]["update_time"] = time.time()
+            return True
+        return False
+
+    def delete_session(self, hash_id: str):
+        if hash_id in self.session:
+            del self.session[hash_id]
+            return True
+        return False

function/__init__.py

@@ -0,0 +1,3 @@
+# from . import topic_clustering_v2
+# from . import sentence_embedding
+from SessionProcess import SessionProcess

function/clean_text.py

@@ -0,0 +1,84 @@
+import re
+
+def normalize_text(text):
+    # text = text.lower()
+    text = text.replace('🏻', '')
+    full_text_clean = text
+    full_text_clean.replace(");this.closest('table').remove();","")
+    full_text_clean = re.sub('(Thứ .{2,4}|Chủ nhật),( ngày)? \d{1,2}\/\d{1,2}\/\d{4}( \d{1,2}:\d{1,2})?( AM| PM)?( \(GMT.{1,3}\))?','',full_text_clean)
+    if not any([word in full_text_clean[:15].lower() for word in ['nga','covid']]):
+        full_text_clean = re.sub('^.{1,15}?-','',full_text_clean)
+    # full_text_clean = re.sub('-.{1,15}?$','',full_text_clean)
+    
+    full_text_clean = re.sub('- VNExplorer','',full_text_clean)
+    full_text_clean = re.sub('Theo .{1,20}$','',full_text_clean)
+    full_text_clean = re.sub('\(.*(Ảnh|Nguồn).*?\)','',full_text_clean)
+    full_text_clean = re.sub('\d{1,2} (giờ|phút) trước','',full_text_clean)
+    full_text_clean = re.sub(r"http\S+", "", full_text_clean)
+    # full_text_clean = re.sub('(\\r)*( )*(\\n)*( )*(\\r)*( )*(\\n)','.', full_text_clean)
+    full_text_clean = re.sub(r"\.( )*(\.)+", '. ', full_text_clean)
+    full_text_clean = re.sub('\.(?!\d)', '. ', full_text_clean)
+    full_text_clean = re.sub('(\.(\s)+)+', '. ', full_text_clean)
+    full_text_clean = re.sub('<[^<]+?>', '',full_text_clean)
+    full_text_clean = re.sub('\d{1,2}:\d{2}( )?\d{1,2}\/\d{1,2}\/\d{4}','',full_text_clean)
+    full_text_clean = re.sub("Ảnh(:)?(Getty)?","", full_text_clean)
+    full_text_clean = full_text_clean.replace("Read more about:","").replace("Read more","").replace("Advertising","").replace("bookmark border.","").replace('the latest tech news, global tech news daily, tech news today, startups, usa tech, asia tech, china tech, eu tech, global tech, in-depth electronics reviews, 24h tech news, 24h tech news, top mobile apps, tech news daily, gaming hardware, big tech news, useful technology tips, expert interviews, reporting on the business of technology, venture capital funding, programing language',"").replace('Live updates:','').replace('-VNExplorer','').replace('Reuters:','').replace('AFP:','').replace('�','').replace('- VNExplorer','').replace('Global Tech News Daily','').replace('AFP/TTXVN','').replace('Reuters/TTXVN','').replace('Tin quốc tế','').replace('Xem tiện ích cảnh báo giá CK','')
+    full_text_clean = full_text_clean.replace("COVID","Covid")
+    full_text_clean = re.sub('[A-Z ]{2,10}(,.{6,20})?(—|–|－|-)','',full_text_clean) #NEW YORK, Feb 27 — .... /BRUSSELS—...
+    full_text_clean = re.sub('\(ảnh:.*?\)','.',full_text_clean)
+    full_text_clean = re.sub("(\| )?(\(.{1,7}\)( )?)+$", "", full_text_clean)
+    full_text_clean = re.sub('\d{2} [\w]{3,4}, \d{4}. \d{2}.\d{2} (AM|PM) IST','',full_text_clean)  #02 Mar, 2022, 10.01 AM IST
+    full_text_clean = full_text_clean.replace('Suzuka. config. supports_premium_subscription && window. localStorage. getItem ( "premiumSubscription ")) ) {var e = document. createElement ( "script "); e. setAttribute ( "class ", "titan-conditional "); e. setAttribute ( "data-ad-id ", "adspot-300x250-pos1 "); document. body. appendChild (e);}','')
+    
+    full_text_clean = re.sub('\d{2}\/\d{2}\/\d{4} \d{2}:\d{2} GMT(\+|-)\d{1,2}', "", full_text_clean)
+    full_text_clean = re.sub('\(.{1,20}\)', '', full_text_clean)
+    full_text_clean = re.sub('\{.{1,20}\}', '', full_text_clean)
+    full_text_clean = re.sub('\[.{1,20}\]', '', full_text_clean)
+    full_text_clean = re.sub('[A-Z].{5,10} , \d{2}:\d{2} (GMT(\+|-)\d{1,2})?',"",full_text_clean)
+
+    full_text_clean = re.sub('(Theo|theo) .{1,15},', '', full_text_clean)
+    full_text_clean = re.sub('(Theo|theo) .{1,15},', '', full_text_clean)
+    full_text_clean = re.sub('theo.{3,20}$','', full_text_clean)
+    full_text_clean = re.sub('^Trong khi đó','', full_text_clean)
+
+    full_text_clean = re.sub('^\d{1,10} minutes ago', '', full_text_clean)
+    full_text_clean = re.sub('^\d{1,10} hours ago', '', full_text_clean)
+    full_text_clean = re.sub('^\d{1,10} days ago', '', full_text_clean)
+    full_text_clean = re.sub('^\d{1,10} years ago', '', full_text_clean)
+    full_text_clean = re.sub('^\d{1,10} months ago', '', full_text_clean)
+    full_text_clean = re.sub('^\d{1,10} minute ago', '', full_text_clean)
+    full_text_clean = re.sub('^\d{1,10} day ago', '', full_text_clean)
+    full_text_clean = re.sub('^\d{1,10} year ago', '', full_text_clean)
+    full_text_clean = re.sub('^\d{1,10} month ago', '', full_text_clean)
+    full_text_clean = re.sub('^\d{1,10} hour ago', '', full_text_clean)
+    full_text_clean = re.sub('^(a|an) minute ago', '', full_text_clean)
+    full_text_clean = re.sub('^(a|an) hour ago', '', full_text_clean)
+    full_text_clean = re.sub('^(a|an) day ago', '', full_text_clean)
+    full_text_clean = re.sub('^(a|an) month ago', '', full_text_clean)
+    full_text_clean = re.sub('^(a|an) year ago', '', full_text_clean)
+    full_text_clean = re.sub('^.{0,12}(tờ|theo|nguồn|trích dẫn|trang|báo|hãng).{1,30}(đưa tin|dẫn l��i|trích dẫn|nhận định|cho biết)', '', full_text_clean, flags=re.I)
+
+    text = re.sub('\s+',' ',full_text_clean)
+    text = re.sub('Đọc chi tiết bài viết tại đây.*','',text,flags=re.I)
+    # text = re.sub('[(\d)(\:)(\|)(\/)(\s+)]+','',text) # 10:20 | 09/03/2022
+    
+    text = re.sub('(\d{1,2}:\d{2}( )*)\|( )*\d{1,2}(/|-)\d{2}(/|-)\d{4}','',text)
+    text = re.sub('^(\d)+[\,\.]\s+ ','',text)   # 3, phát ngôn viên Bộ T
+    text = re.sub('((chủ nhật)|(thứ bảy)|(thử sáu)|(thứ năm)|(thứ tư)|(thứ ba)|(thứ hai))([(\d)(\:)(,)(\|\/)(\s+)]+)((VOV)|(VTV))$','',text,flags=re.I) #   và Ukraine để giải quyết xung đột Chủ Nhật, 06:32, 20/03/2022 VOV.
+
+    text = re.sub('^((\d)|(\:)|(\.)|(\|)|(\s+)|(in bài biết)|(in bài viết)|(\/))+ ','',text,flags=re.I) # 10:20 | 09/03/2022 In bài biết. 10:20 | 09/03/2022 In bài biết Việc xuất khẩu tôm sang thị trường Nga có thể bị ảnh hưởng trước tình hình căng thẳng của Nga-Ukraine. Hiệp hội Chế biến và Xuất khẩu thuỷ sản V
+    text = re.sub('theo hãng tin [A-Z].{0,15},','', text, flags=re.I)
+    text = re.sub('((Theo.{0,30})|(Reuters)).*?(link gốc).*?$','',text,flags=re.I) 
+    text = re.sub('video:RIA Novosti/Bộ Quốc phòng Nga','',text,flags=re.I)
+    text = re.sub('Báo.{5,20}$','',text)
+    text = re.sub('RIA Novosti/Bộ Quốc phòng Nga','',text)
+    text = re.sub('(chính trị|quân sự|đối ngoại|thời sự|khoa học|pháp luật) \d{1,2} giờ','',text,flags=re.I)
+    text = text.replace('|','')
+    full_text_clean = re.sub('^.*?(Link nguồn)','',text,flags=re.I) # (
+    full_text_clean = re.sub(',( )*[A-z].{1,30}(đưa tin|trích dẫn)','', full_text_clean)
+    full_text_clean = re.sub('(Reuters|Vnexpress)(\).)?','',full_text_clean,flags=re.I) 
+    full_text_clean = re.sub('^VOV.','',full_text_clean) 
+    full_text_clean = full_text_clean.replace("Many Japanese worry Fortune-Takashi Nakamichi, Bloomberg • 1d","").replace('baotintuc. vn',"").replace('YÊN BÁI QUYẾT TÂM THỰC HIỆN THẮNG LỢI CHƯƠNG TRÌNH HÀNH ĐỘNG SỐ 56 – CTr/TU CỦA TỈNH ỦY QUYẾT TÂM ĐƯA Y.ÊN BÁI PHÁT TRIỂN NHANH, BỀN VỮNG THEO HƯỚNG “XANH, HÀI HÒA, BẢN SẮC VÀ HẠNH PHÚC” TRỞ THÀNH TỈNH PHÁT TRIỂN KHÁ VÀO NĂM 2025','')
+    full_text_clean = full_text_clean.replace("Baoquocte","").replace('ชั่วโมงที่ผ่านมา.','').replace('""challenge=" "coron= ""corona=" "covid-19= ""designs=" "endgame= ""tutorial=" "ui= ""pandemic=" "quarantine= ""list=" "similarity= " "ux. ""press=" "copyright= ""contact=" "creators= ""advertise=" "terms= ""privacy=" "policy= ""safety=" "youtube= ""works=" "test= ""features=" "creators.', '').replace('nbsp & nbsp & nbsp & nbsp & nbsp Copy Link', '').replace('Tổng thống Nga Vladimir Putin và Chủ tịch Trung Quốc Tập Cận Bình.','').replace('Thế giới Toàn cảnh Bảo Hà ','')
+    full_text_clean = re.sub('(a|p)\. m\.','',full_text_clean) 
+    return full_text_clean

function/detect_time.py

@@ -0,0 +1,92 @@
+import re
+import requests
+import datetime
+import operator
+from typing import *
+from dateutil.relativedelta import *
+from itertools import groupby
+from dateparser import parse
+
+day = '[0-3]{0,1}[0-9]'
+month = '[0,1]{0,1}[0-9]'
+year = '\d{4}'
+sep = '\s*[-/\.\s]\s*'
+
+patterns = [
+    f"{day}{sep}{month}{sep}{year}",
+    f"{year}{sep}{month}{sep}{day}",
+    f"{day}\s+tháng\s+{month}",
+    f"{day}\s+tháng\s+{month}{sep}{year}",
+    f"{day}\s+tháng\s+{month}\s+năm\s+{year}",
+    f"{day}\s+tháng\s+{month}",
+    f"(?<=ngày)\s+{day}{sep}{month}",
+    f"(?<=ngày)\s+{day}{sep}{month}{sep}{year}",
+    f"(?<=sáng)\s+{day}{sep}{month}",
+    f"(?<=sáng)\s+{day}{sep}{month}{sep}{year}",
+    f"(?<=trưa)\s+{day}{sep}{month}",
+    f"(?<=trưa)\s+{day}{sep}{month}{sep}{year}",
+    f"(?<=chiều)\s+{day}{sep}{month}",
+    f"(?<=chiều)\s+{day}{sep}{month}{sep}{year}",
+    f"(?<=tối)\s+{day}{sep}{month}",
+    f"(?<=tối)\s+{day}{sep}{month}{sep}{year}"
+    f"(?<=đêm)\s+{day}{sep}{month}",
+    f"(?<=đêm)\s+{day}{sep}{month}{sep}{year}",
+    f"(?<=hôm)\s+{day}{sep}{month}",
+    f"(?<=hôm)\s+{day}{sep}{month}{sep}{year}",
+    f"{day}{sep}{month}[\s\.\,\)]"
+]
+
+
+def extract_pattern(text: str, patterns: List[str]):
+    detected = []
+    for pattern in patterns:
+        for match in re.finditer(pattern, text):
+            detected.append((match.start(), match.end()))
+    detected.sort()
+    output = []
+    curr = -1
+    for start, values in groupby(detected, key=operator.itemgetter(0)):
+        if start < curr:
+            continue
+        values = list(values)
+        values.sort(key=operator.itemgetter(1), reverse=True)
+        output.append(values[0])
+        curr = values[0][1]
+    return output
+
+
+def detect_time(text: str, language: str = 'vi', base: Optional[datetime.datetime] = None):
+    text = text.lower()
+    detected_patterns = extract_pattern(text, patterns)
+    output = []
+    settings = {
+        'PREFER_DAY_OF_MONTH': 'first'
+    }
+    if base:
+        settings['RELATIVE_BASE'] = base
+    for start, end in detected_patterns:
+        segment = text[start:end]
+        segment = re.sub('\s+', ' ', segment).strip().lower()
+        candiate = parse(segment, languages=[language], settings=settings)
+        output.append((segment, candiate))
+    return output
+
+
+def get_time_post(sentences, patterns, start_time=None, end_time=None):
+    dict_time_evs = {}
+    for i, sen in enumerate(sentences):
+        if sen.strip() != "":
+            time_ex = detect_time(sen, patterns)
+            for te in time_ex:
+                if te[1] is not None:
+                    if start_time is None or end_time is None or (end_time > te[1].timestamp() > start_time):
+                        if te not in dict_time_evs:
+                            dict_time_evs[te] = []
+                        dict_time_evs[te].append(i)
+    return dict_time_evs
+
+
+if __name__ == '__main__':
+    print(detect_time("VietTimes – Ngoại trưởng Mỹ Antony Blinken ngày đã tuyên bố trong một cuộc họp qua"
+                      "truyền hình với ngoại trưởng các nước ASEAN Mỹ bác bỏ các yêu sách “bất hợp pháp” của"
+                      "Trung Quốc ở Biển Đông.", language="vi"))

function/embed_vncorenlp.py

@@ -0,0 +1,161 @@
+import jnius_config
+import os
+import shutil
+
+save_dir = "/home2/vietle/icgpt/vncorenlp-1.2"
+max_heap_size='-Xmx4g'
+jnius_config.add_options(max_heap_size)
+jnius_config.set_classpath(save_dir + "/VnCoreNLP-1.2.jar")
+
+def download_model(save_dir='./'):
+    # current_path = os.path.abspath(os.getcwd())
+    if save_dir[-1] == '/':
+        save_dir = save_dir[:-1]
+    if os.path.isdir(save_dir + "/models") and os.path.exists(save_dir + '/VnCoreNLP-1.2.jar'):
+        print("VnCoreNLP model folder " + save_dir + " already exists! Please load VnCoreNLP from this folder!")
+    else:
+        os.mkdir(save_dir + "/models")
+        os.mkdir(save_dir + "/models/dep")
+        os.mkdir(save_dir + "/models/ner")
+        os.mkdir(save_dir + "/models/postagger")
+        os.mkdir(save_dir + "/models/wordsegmenter")
+        # jar
+        os.system("wget https://raw.githubusercontent.com/vncorenlp/VnCoreNLP/master/VnCoreNLP-1.2.jar")
+        shutil.move("VnCoreNLP-1.2.jar", save_dir + "/VnCoreNLP-1.2.jar")
+        # wordsegmenter
+        os.system("wget https://raw.githubusercontent.com/vncorenlp/VnCoreNLP/master/models/wordsegmenter/vi-vocab")
+        os.system(
+            "wget https://raw.githubusercontent.com/vncorenlp/VnCoreNLP/master/models/wordsegmenter/wordsegmenter.rdr")
+        shutil.move("vi-vocab", save_dir + "/models/wordsegmenter/vi-vocab")
+        shutil.move("wordsegmenter.rdr", save_dir + "/models/wordsegmenter/wordsegmenter.rdr")
+        # postagger
+        os.system("wget https://raw.githubusercontent.com/vncorenlp/VnCoreNLP/master/models/postagger/vi-tagger")
+        shutil.move("vi-tagger", save_dir + "/models/postagger/vi-tagger")
+        # ner
+        os.system("wget https://raw.githubusercontent.com/vncorenlp/VnCoreNLP/master/models/ner/vi-500brownclusters.xz")
+        os.system("wget https://raw.githubusercontent.com/vncorenlp/VnCoreNLP/master/models/ner/vi-ner.xz")
+        os.system(
+            "wget https://raw.githubusercontent.com/vncorenlp/VnCoreNLP/master/models/ner/vi-pretrainedembeddings.xz")
+        shutil.move("vi-500brownclusters.xz", save_dir + "/models/ner/vi-500brownclusters.xz")
+        shutil.move("vi-ner.xz", save_dir + "/models/ner/vi-ner.xz")
+        shutil.move("vi-pretrainedembeddings.xz", save_dir + "/models/ner/vi-pretrainedembeddings.xz")
+        # parse
+        os.system("wget https://raw.githubusercontent.com/vncorenlp/VnCoreNLP/master/models/dep/vi-dep.xz")
+        shutil.move("vi-dep.xz", save_dir + "/models/dep/vi-dep.xz")
+
+
+class VnCoreNLP:
+    def __init__(self, annotators=["wseg", "pos", "ner", "parse"], save_dir = './'):
+        if save_dir[-1] == '/':
+            save_dir = save_dir[:-1]
+        if os.path.isdir(save_dir + "/models") == False or os.path.exists(save_dir + '/VnCoreNLP-1.2.jar') == False:
+            raise Exception("Please download the VnCoreNLP model!")
+        self.current_working_dir = os.getcwd()
+        os.chdir(save_dir)
+       
+        from jnius import autoclass
+        javaclass_vncorenlp = autoclass('vn.pipeline.VnCoreNLP')
+        self.javaclass_String = autoclass('java.lang.String')
+        self.annotators = annotators
+        if "wseg" not in annotators:
+            self.annotators.append("wseg")
+
+        self.model = javaclass_vncorenlp(annotators)
+
+    def annotate_text(self, text):
+        from jnius import autoclass
+        javaclass_Annotation = autoclass('vn.pipeline.Annotation')
+        str = self.javaclass_String(text)
+        annotation = javaclass_Annotation(str)
+        self.model.annotate(annotation)
+        dict_sentences = {}
+        list_sentences = annotation.toString().split("\n\n")[:-1]
+        for i in range(len(list_sentences)):
+            list_words = list_sentences[i].split("\n")
+            list_dict_words = []
+            for word in list_words:
+                dict_word = {}
+                word = word.replace("\t\t", "\t")
+                list_tags = word.split("\t")
+                dict_word["index"] = int(list_tags[0])
+                dict_word["wordForm"] = list_tags[1]
+                dict_word["posTag"] = list_tags[2]
+                dict_word["nerLabel"] = list_tags[3]
+                if "parse" in self.annotators:
+                    dict_word["head"] = int(list_tags[4])
+                else:
+                    dict_word["head"] = list_tags[4]
+                dict_word["depLabel"] = list_tags[5]
+                list_dict_words.append(dict_word)
+            dict_sentences[i] = list_dict_words
+        return dict_sentences
+
+    def tokenize(self, text):
+        annotated_sens = self.annotate_text(text=text)
+        output = []
+        for id_sen in annotated_sens:
+            annotated_sen = annotated_sens[id_sen]
+            out = [s["wordForm"] for s in annotated_sen]
+            output.append(out)
+        return output
+
+    def pos_tag(self, text):
+        annotated_sens = self.annotate_text(text=text)
+        output = []
+        for id_sen in annotated_sens:
+            annotated_sen = annotated_sens[id_sen]
+            out = [(s["wordForm"], s["posTag"]) for s in annotated_sen]
+            output.append(out)
+        return output
+
+    def ner(self, text):
+        annotated_sens = self.annotate_text(text=text)
+        output = []
+        for id_sen in annotated_sens:
+            annotated_sen = annotated_sens[id_sen]
+            out = [(s["wordForm"], s["nerLabel"]) for s in annotated_sen]
+            output.append(out)
+        return output
+
+    def word_segment(self, text):
+        from jnius import autoclass
+        javaclass_Annotation = autoclass('vn.pipeline.Annotation')
+        str = self.javaclass_String(text)
+        annotation = javaclass_Annotation(str)
+        self.model.annotate(annotation)
+        list_segmented_sentences = []
+        list_sentences = annotation.toString().split("\n\n")[:-1]
+        for sent in list_sentences:
+            list_words = sent.split("\n")
+            list_segmented_words = []
+            for word in list_words:
+                word = word.replace("\t\t", "\t")
+                list_tags = word.split("\t")
+                list_segmented_words.append(list_tags[1])
+            list_segmented_sentences.append(" ".join(list_segmented_words))
+        return list_segmented_sentences
+
+    def print_out(self, dict_sentences):
+        for sent in dict_sentences.keys():
+            list_dict_words = dict_sentences[sent]
+            for word in list_dict_words:
+                print(str(word["index"]) + "\t" + word["wordForm"] + "\t" + word["posTag"] + "\t" + word["nerLabel"] + "\t" + str(word["head"]) + "\t" + word["depLabel"])
+            print("")
+
+    def annotate_file(self, input_file, output_file):
+        os.chdir(self.current_working_dir)
+        input_str = self.javaclass_String(input_file)
+        output_str = self.javaclass_String(output_file)
+        self.model.processPipeline(input_str, output_str, self.annotators)
+
+if __name__ == '__main__':
+    download_model(save_dir='/home2/vietle/icgpt/vncorenlp-1.2')
+    model = VnCoreNLP(annotators=["wseg","pos","ner"], save_dir='/home2/vietle/icgpt/vncorenlp-1.2')
+    # output = model.annotate_text("Ông Nguyễn Khắc Chúc  đang làm việc tại Đại học Quốc gia Hà Nội. Bà Lan, vợ ông Chúc, cũng làm việc tại đây.")
+    # print(output)
+
+    text = "Sau khi tốt nghiệp Trung học năm 1975, ông theo học dự bị Ngoại ngữ tại Đại học Ngoại ngữ (nay là Trường Đại học Hà Nội)."
+    out = model.tokenize(text)
+    print(out)
+    # model.print_out(output)
+    

function/sentence_embbeding.py

@@ -0,0 +1,41 @@
+import requests
+import json
+
+URL_EMBBED_ZH = "http://10.9.3.239:1999/api/v1/extract_feature_zh"
+URL_EMBBED_EN = "http://10.9.3.239:1999/api/v1/extract_feature_en"
+URL_EMBBED_BGE = "http://10.9.3.240:5045/api/v1/embedding"
+
+def embbeded_zh(text: list):
+    try:
+        r = requests.post(URL_EMBBED_ZH, data = json.dumps({
+            "text": text
+        }))
+        embs = r.json()["vectors"]
+        return embs
+    except Exception as ve:
+        print(ve)
+        return []
+
+
+def embbeded_en(text: list):
+    try:
+        r = requests.post(URL_EMBBED_EN, data = json.dumps({
+            "text": text
+        }))
+        embs = r.json()["vectors"]
+        return embs
+    except Exception as ve:
+        print(ve)
+        return []
+    
+
+def embedded_bge(text: list):
+    try:
+        r = requests.post(URL_EMBBED_BGE, data = json.dumps({
+            "text": text
+        }))
+        embs = r.json()["embeddings"]
+        return embs
+    except Exception as ve:
+        print(ve)
+        return []

function/summary_with_llm.py

@@ -0,0 +1,210 @@
+import json
+import requests
+import nltk
+import re
+import time
+
+nltk.download('punkt')
+
+
+URL_LLAMA = "http://10.9.3.241:8022/api/v1/llama/QnA"
+VERSION = {
+    "en-llama": "mistral-full", #"llama2-7b",
+    "vi-llama": "mistral-full" #"ic-llama-68k"
+}
+
+SYSTEM_PROMPT = ""
+POST_FIX_Q = ""
+
+SYSTEM_PROMPT_CHECK_RELEVANT = "Bạn là trợ lý AI giúp mọi người tìm kiếm thông tin. Người dùng sẽ cung cấp cho bạn một câu hỏi. Nhiệm vụ của bạn là trả lời trung thực nhất có thể."
+PROMPT_RELEVANT = 'Câu trả lời dưới có liên quan đến câu hỏi "{0}" hay không?'
+
+
+REMOVE_WORDS = []
+with open("data/remove.txt", 'r') as f:
+    for line in f:
+        line = line.strip()
+        if line:
+            REMOVE_WORDS.append(line.lower())
+print(REMOVE_WORDS)
+
+
+def check_answer_contains_remove_words(text):
+    text = text.lower()
+    text = re.sub(r'\s+ ', ' ', text)
+    lst_sen = nltk.sent_tokenize(text)
+    for sen in lst_sen:
+        for pat in REMOVE_WORDS:
+            if re.search(pat, sen) is not None:
+                return True
+    return False
+
+
+def normalize_text_after_qna_llm(text, prompt: str = ""):
+
+    text = re.sub("^Dựa trên.{1,60}cung cấp,", "", text.strip(), flags=re.I).strip()
+    text = re.sub("^Dự báo.{1,60}là", "", text, flags=re.I).strip()
+    text = re.sub("^.{1,15}dựa trên.{1,60}cung cấp,", "", text, flags=re.I).strip()
+    text = re.sub("^.{1,15}theo thông tin.{1,60}cung cấp,", "", text, flags=re.I).strip()
+    # if len(prompt) > 10:
+    #     print("pattern: ", "^" + prompt[:10] + ".{1,40}là")
+    #     text = re.sub("^" + prompt[:10] + ".{1,40}là", "", text, flags=re.I).strip()
+    if text and text[0].islower():
+        text = text[0].upper() + text[1:]
+    return text
+
+
+def created_context(text, max_word=2048):
+    text = re.sub(r'\s+ ', ' ', text)
+    lst_sen = nltk.sent_tokenize(text)
+    count_w = 0
+    contexts = []
+    lst_sen_per_context = []
+    for sen in lst_sen:
+        count_w += sen.count(" ")
+        lst_sen_per_context.append(sen)
+        if count_w > max_word:
+            contexts.append(" ".join(lst_sen_per_context))
+            lst_sen_per_context = []
+            count_w = 0
+    if lst_sen_per_context:
+        contexts.append(" ".join(lst_sen_per_context))
+    return contexts
+
+
+def get_icgpt_result(questions, contexts, lang, version="vi-llama", system_prompt:str = "", id_cluster="", hash_str: str = ""):
+    json_body = {
+        "questions": questions,
+        "contexts": contexts,
+        "lang": lang,
+        "version": VERSION[version],
+        "system_prompt": system_prompt
+    }
+    try:
+        
+        res = requests.post(URL_LLAMA, json=json_body)
+        with open(f"log_llm/requests_llama/{hash_str}_{id_cluster}.txt", "w") as f:
+            f.write(json.dumps(json_body) + "\n")
+            f.write(json.dumps(res.json()) + "\n" + "$"*50)
+        res = res.json()["result"]
+        res = [x.replace("_", " ") for x in res]
+        return res
+    except Exception as ex:
+        print(f"[ERROR] get_icgpt_official_visit: {str(ex)}")
+        with open(f"log_llm/requests_llama_error/{hash_str}_{id_cluster}.txt", "w") as f:
+            f.write(json.dumps(json_body) + "\n")
+        return None
+
+
+def check_relevant_with_llama(question, content, lang, version="en-llama", max_word_per_context=1024, id_cluster="", hash_str: str = ""):
+    contexts = created_context(content, max_word=max_word_per_context)
+    content = contexts[0] if contexts else ""
+    answer = ""
+    question_all = question.split("#####")
+    check_rel = True
+    for question in question_all:
+        question = question.strip()
+        # question = PROMPT_RELEVANT.format(question)
+        question_split = question.split("$$$$")
+        if len(question_split) < 2:
+            question = question
+            system_prompt = ""
+        else:
+            system_prompt = question_split[0].strip()
+            question = question_split[1].strip()
+
+        # system_prompt = SYSTEM_PROMPT_CHECK_RELEVANT
+        if content:
+            contexts = [content]
+            questions = [question]
+            lst_res = get_icgpt_result(questions, contexts, lang, version=version, system_prompt=system_prompt, id_cluster=f"{id_cluster}_recheck_relevant", hash_str=hash_str)
+            if lst_res is None:
+                lst_res = []
+                check_rel = True
+                # continue
+                # break
+            for an in lst_res:
+                if an:
+                    is_relevant = check_answer_contains_remove_words(an)
+                    answer = normalize_text_after_qna_llm(an, prompt=question)
+                    if len(answer) == 0:
+                        check_rel = True
+                        # continue
+                        # break
+                    else:
+                        check_rel = is_relevant
+                        if not check_rel:
+                            break
+            if not check_rel:
+                break
+        else:
+            check_rel = True
+    return check_rel
+
+
+def summary_with_llama(question, content, lang, version="vi-llama", max_word_per_context=1024, id_cluster="", hash_str: str = ""):
+    contexts = created_context(content, max_word=max_word_per_context)
+    question_split = question.split("$$$$")
+    if len(question_split) < 2:
+        question = question
+        system_prompt = ""
+    else:
+        system_prompt = question_split[0].strip()
+        question = question_split[1].strip()
+
+    if question.strip().endswith(".") or question.strip().endswith("?"):
+        question = (question + POST_FIX_Q).strip()
+    else:
+        question = (question + ". " + POST_FIX_Q).strip()
+    answer = ""
+    if contexts:
+        contexts = [contexts[0]]
+        questions = [question] * len(contexts)
+        lst_res = get_icgpt_result(questions, contexts, lang, version=version, system_prompt=system_prompt, id_cluster=id_cluster, hash_str=hash_str)
+        results = []
+        if lst_res is None:
+            lst_res = []
+        for an in lst_res:
+            
+            is_relevant = check_answer_contains_remove_words(an)
+            
+            if not is_relevant:
+                results.append(an)
+            # else:
+            #     print("ans_from_llama-preview: ", an, is_relevant)
+        if results:
+            if len(results) > 1:
+                context_combine = [". ".join(results)]
+                res = get_icgpt_result([question], context_combine, lang, version=version, system_prompt=system_prompt, id_cluster=id_cluster, hash_str=hash_str)
+                if res:
+                    answer = res[0]
+            else:
+                answer = results[0]
+        else:
+            if lst_res:
+                return "", True
+    if answer:
+        is_relevant = check_answer_contains_remove_words(answer)
+    # print("ans_from_llama-before: ", answer, is_relevant)
+    answer = normalize_text_after_qna_llm(answer, prompt=question)
+    
+    if len(answer) == 0:
+        is_relevant = True
+    # print("ans_from_llama-after: ", answer, is_relevant)
+    return answer, is_relevant
+
+
+if __name__ == '__main__':
+    text = """ Dựa trên thông tin được cung cấp, xung đột ở Biển Đông đã gia tăng và có thể xuất hiện điểm xung đột mới giữa Philippines và Trung Quốc. Philippines đang xem xét xây dựng một ngọn hải đăng trên Bãi cạn Second Thomas (còn gọi là Bãi cạn Second Thomas), điều này đã khiến Trung Quốc tuyên bố rằng đây là một hành động vi phạm nghiêm trọng chủ quyền của họ và sẽ đáp trả kiên quyết. Giới chuyên gia cho rằng điểm bùng phát xung đột mới có thể xảy ra giữa Philippines và Trung nước ở Biển Đông. Tham vọng hung hăng của Trung Quốc đã thúc đẩy căng thẳng khu vực và các tranh chấp gần đây về vụ va chạm gần đây của các tàu gần Bãi Cỏ Mây (Second Thomas Shoal) ở Biển Đông đã làm căng th  ng giữa Trung Quốc và Philippines gia tăng. """
+    is_re = check_answer_contains_remove_words(text)
+    print(is_re)
+    an = normalize_text_after_qna_llm(text)
+    print(an)
+#     exit(1)
+#     message = "G\u1EA7n \u0111\u00E2y, Philippines \u0111\u00E3 nhi\u1EC1u l\u1EA7n g\u00E2y ra r\u1EAFc r\u1ED1i \u1EDF Bi\u1EC3n \u0110\u00F4ng, x\u00E2m nh\u1EADp \u0110\u00E1 Nh\u00E2n \u00C1i thu\u1ED9c qu\u1EA7n \u0111\u1EA3o Nam Sa c\u1EE7a Trung Qu\u1ED1c, \u0111\u1ED3ng th\u1EDDi ti\u1EBFp t\u1EE5c ph\u00F3ng \u0111\u1EA1i v\u00E0 c\u01B0\u1EDDng \u0111i\u1EC7u h\u00F3a, nh\u1EA7m l\u1EABn \u0111\u00FAng sai v\u00E0 \u0111\u01B0a ra nh\u1EEFng c\u00E1o bu\u1ED9c v\u00F4 l\u00FD ch\u1ED1ng l\u1EA1i Trung Qu\u1ED1c. \\n Ng\u01B0\u1EDDi Philippines ngh\u0129 g\u00EC v\u1EC1 h\u00E0nh \u0111\u1ED9ng c\u1EE7a ch\u00EDnh ph\u1EE7 Philippines? \\ n \u00D4ng Tan \u0111\u00E3 t\u00ECm ra Herman Laurel, m\u1ED9t nh\u00E0 b\u00ECnh lu\u1EADn ch\u00EDnh tr\u1ECB n\u1ED5i ti\u1EBFng \u1EDF Philippines, \u0111\u1ED3ng th\u1EDDi l\u00E0 gi\u00E1m \u0111\u1ED1c Vi\u1EC7n nghi\u00EAn c\u1EE9u chi\u1EBFn l\u01B0\u1EE3c \"Th\u1EBF k\u1EF7 ch\u00E2u \u00C1\" c\u1EE7a Philippines v\u00E0 l\u00E0 ng\u01B0\u1EDDi s\u00E1ng l\u1EADp Hi\u1EC7p h\u1ED9i nghi\u00EAn c\u1EE9u ch\u00EDnh s\u00E1ch BRICS c\u1EE7a Philippines. Herman Laurel, ng\u01B0\u1EDDi \u0111ang nghi\u00EAn c\u1EE9u v\u1EC1 ch\u00E2u \u00C1, \u0111\u1EB7c bi\u1EC7t l\u00E0 m\u1ED1i quan h\u1EC7 gi\u1EEFa Trung Qu\u1ED1c v\u00E0 ASEAN, chia s\u1EBB quan s\u00E1t c\u1EE7a m\u00ECnh. \\n 01 \\n Tan Zhu: K\u1EC3 t\u1EEB n\u1EEDa cu\u1ED1i n\u0103m nay, Philippines th\u01B0\u1EDDng xuy\u00EAn x\u00E2m chi\u1EBFm c\u00E1c \u0111\u1EA3o v\u00E0 b\u00E3i \u0111\u00E1 c\u1EE7a ch\u00FAng t\u00F4i \u1EDF Bi\u1EC3n \u0110\u00F4ng, l\u00E0m gia t\u0103ng c\u0103ng th\u1EB3ng \u1EDF Bi\u1EC3n \u0110\u00F4ng, \u0111\u1ED3ng th\u1EDDi vu kh\u1ED1ng Trung Qu\u1ED1c l\u00E0 k\u1EBB b\u1EAFt n\u1EA1t l\u1EDBn. , c\u00F3 nh\u1EEFng nh\u00F3m \u1EDF Philippines ph\u1EA3n \u0111\u1ED1i h\u00E0nh \u0111\u1ED9ng khi\u00EAu kh\u00EDch c\u1EE7a Philippines. B\u1EA1n c\u1EA3m th\u1EA5y th\u1EBF n\u00E0o v\u1EC1 \u0111i\u1EC1u n\u00E0y v\u1EDBi t\u01B0 c\u00E1ch l\u00E0 m\u1ED9t ng\u01B0\u1EDDi Philippines? Herman Laurel: T\u00F4i cho r\u1EB1ng C\u1EA3nh s\u00E1t bi\u1EC3n Philippines v\u00E0 m\u1ED9t s\u1ED1 t\u00E0u d\u00E2n s\u1EF1 Philippines \u0111\u00E3 c\u1ED1 t\u00ECnh g\u00E2y r\u1EAFc r\u1ED1i v\u00EC h\u1ECD \u0111ang th\u1EF1c hi\u1EC7n \u201CD\u1EF1 \u00E1n t\u1ED5ng th\u1EC3\u201D c\u1EE7a Hoa K\u1EF3. \\nM\u1EF9 khoe khoang d\u1EF1 \u00E1n n\u00E0y v\u1EDBi ch\u00FAng t\u00F4i n\u00EAn ch\u00FAng t\u00F4i bi\u1EBFt t\u00EAn d\u1EF1 \u00E1n. D\u1EF1 \u00E1n \u0111\u01B0\u1EE3c d\u1EABn d\u1EAFt b\u1EDFi \u0110\u1EA1i t\u00E1 Kh\u00F4ng qu\u00E2n Hoa K\u1EF3 \u0111\u00E3 ngh\u1EC9 h\u01B0u Raimundo Powell, l\u00E0m vi\u1EC7c v\u1EDBi c\u00E1c \u0111\u1ED1i t\u00E1c Philippines nh\u01B0 C\u1EA3nh s\u00E1t bi\u1EC3n Philippines v\u00E0 m\u1ED9t s\u1ED1 quan ch\u1EE9c c\u1EE7a B\u1ED9 Ngo\u1EA1i giao Philippines. H\u1ECD l\u00EAn k\u1EBF ho\u1EA1ch, \u00E2m m\u01B0u v\u00E0 th\u1EF1c hi\u1EC7n d\u1EF1 \u00E1n n\u00E0y nh\u1EB1m \u0111\u1ED1i \u0111\u1EA7u, khi\u00EAu kh\u00EDch v\u00E0 \u0111\u1ED1i \u0111\u1EA7u v\u1EDBi Qu\u00E2n \u0111o\u00E0n C\u1EA3nh s\u00E1t bi\u1EC3n thu\u1ED9c L\u1EF1c l\u01B0\u1EE3ng C\u1EA3nh s\u00E1t V\u0169 trang Nh\u00E2n d\u00E2n Trung Qu\u1ED1c. T\u1EA5t c\u1EA3 nh\u1EEFng \u0111i\u1EC1u n\u00E0y \u0111\u1EC1u nh\u1EB1m m\u1EE5c \u0111\u00EDch c\u1ED1 t\u00ECnh t\u1EA1o ra c\u0103ng th\u1EB3ng \u1EDF Bi\u1EC3n \u0110\u00F4ng, \u0111\u1EB7c bi\u1EC7t l\u00E0 t\u1EA1i c\u00E1c khu v\u1EF1c tranh ch\u1EA5p gi\u1EEFa Philippines v\u00E0 Trung Qu\u1ED1c nh\u01B0 B\u00E3i c\u1EA1n Scarborough hay B\u00E3i c\u1EA1n Second Thomas. \\n \\n \\n \\n \\n Vi\u1EC7c Trung Qu\u1ED1c s\u1EED d\u1EE5ng t\u00E0u l\u1EDBn v\u00E0 t\u00E0u chi\u1EBFn l\u00E0 \u0111i\u1EC1u t\u1ED1t. L\u1EE3i th\u1EBF v\u1EC1 s\u1ED1 l\u01B0\u1EE3ng v\u00E0 quy m\u00F4 c\u1EE7a L\u1EF1c l\u01B0\u1EE3ng C\u1EA3nh s\u00E1t bi\u1EC3n thu\u1ED9c L\u1EF1c l\u01B0\u1EE3ng C\u1EA3nh s\u00E1t V\u0169 trang Nh\u00E2n d\u00E2n Trung Qu\u1ED1c th\u1EF1c s\u1EF1 tr\u00E1nh \u0111\u01B0\u1EE3c m\u1ED9t cu\u1ED9c \u0111\u1ED1i \u0111\u1EA7u th\u1EF1c s\u1EF1 nghi\u00EAm tr\u1ECDng h\u01A1n, b\u1EDFi v\u00EC n\u1EBFu hai b\u00EAn c\u00F3 quy m\u00F4 ngang nhau, M\u1ED9t cu\u1ED9c xung \u0111\u1ED9t h\u1EA1m \u0111\u1ED9i c\u00F3 th\u1EC3 tr\u1EDF n\u00EAn s\u1EAFp x\u1EA3y ra. \\ n Nh\u01B0ng d\u00F9 th\u1EBF n\u00E0o \u0111i n\u1EEFa, c\u00E1c ph\u01B0\u01A1ng ti\u1EC7n truy\u1EC1n th\u00F4ng ph\u01B0\u01A1ng T\u00E2y s\u1EBD m\u00F4 t\u1EA3 Trung Qu\u1ED1c l\u00E0 k\u1EBB b\u1EAFt n\u1EA1t. \u0110\u00E2y l\u00E0 m\u1EE5c \u0111\u00EDch th\u1EF1c s\u1EF1 c\u1EE7a d\u1EF1 \u00E1n n\u00E0y v\u00E0 l\u00E0 m\u1EE5c ti\u00EAu m\u00E0 Hoa K\u1EF3 \u0111\u1EB7t ra - h\u1ECD s\u1EED d\u1EE5ng c\u00E1i g\u1ECDi l\u00E0 \"ni\u1EC1m tin v\u00E0o t\u00EDnh minh b\u1EA1ch c\u1EE7a b\u00E1o c\u00E1o\" \u0111\u1EC3 c\u1EA5u th\u00E0nh m\u1ED9t \"\u00E2m m\u01B0u \u0111en t\u1ED1i\" ch\u1ED1ng l\u1EA1i Trung Qu\u1ED1c... c\u00F4ng khai\". \u0110\u01B0\u1EE3c cho l\u00E0 minh b\u1EA1ch v\u00EC truy\u1EC1n th\u00F4ng ph\u01B0\u01A1ng T\u00E2y \u0111\u00E3 mang theo camera, nh\u00E2n vi\u00EAn truy\u1EC1n th\u00F4ng... \u0111\u1EC3 ghi l\u1EA1i qu\u00E1 tr\u00ECnh n\u00E0y v\u00E0 \u0111\u01B0a tin cho kh\u00E1n gi\u1EA3 Philippines v\u00E0 th\u1EBF gi\u1EDBi. \\nHoa K\u1EF3 \u0111\u00E3 ch\u1EC9 ra trong \u0111\u1EC1 xu\u1EA5t d\u1EF1 \u00E1n r\u1EB1ng m\u1EE5c ti\u00EAu c\u1EE7a h\u1ECD l\u00E0 khi\u1EBFn Trung Qu\u1ED1c ph\u1EA3i tr\u1EA3 gi\u00E1. T\u00F4i ngh\u0129 m\u1ECDi ng\u01B0\u1EDDi \u0111\u1EC1u c\u00F3 th\u1EC3 hi\u1EC3u \u0111i\u1EC1u n\u00E0y c\u00F3 ngh\u0129a l\u00E0 g\u00EC - Hoa K\u1EF3 mu\u1ED1n h\u1EE7y ho\u1EA1i danh ti\u1EBFng c\u1EE7a Trung Qu\u1ED1c. \\n C\u00F3 m\u1ED9t th\u1EDDi, Philippines c\u00F3 m\u1ED1i quan h\u1EC7 r\u1EA5t h\u1EEFu \u00EDch v\u00E0 th\u00E2n thi\u1EC7n v\u1EDBi Trung Qu\u1ED1c. C\u00F3 nhi\u1EC1u c\u1EA7u n\u1ED1i h\u1EE3p t\u00E1c gi\u1EEFa ch\u00FAng ta, ch\u1EB3ng h\u1EA1n nh\u01B0 c\u00E1c d\u1EF1 \u00E1n h\u1EE3p t\u00E1c trong vi\u1EC7c c\u00F9ng x\u00E2y d\u1EF1ng s\u00E1ng ki\u1EBFn \u200B\u200B\u201CV\u00E0nh \u0111ai v\u00E0 Con \u0111\u01B0\u1EDDng\u201D, gi\u00FAp t\u0103ng th\u00EAm l\u1EE3i th\u1EBF kinh t\u1EBF c\u1EE7a Philippines. \\n Tuy nhi\u00EAn, T\u1ED5ng th\u1ED1ng Philippines Marcos r\u00F5 r\u00E0ng c\u00F3 quan \u0111i\u1EC3m ri\u00EAng v\u1EC1 s\u1EF1 d\u00E0n x\u1EBFp c\u1EE7a M\u1EF9, \u00F4ng ch\u01B0a bao gi\u1EDD gi\u1EA3i th\u00EDch cho ng\u01B0\u1EDDi d\u00E2n Philippines l\u00FD do quay tr\u1EDF l\u1EA1i v\u1EDBi M\u1EF9. V\u1EC1 v\u1EA5n \u0111\u1EC1 Bi\u1EC3n \u0110\u00F4ng, ch\u00EDnh s\u00E1ch c\u1EE7a Marcos c\u00E0ng th\u00F9 \u0111\u1ECBch v\u00E0 hung h\u0103ng h\u01A1n. \\n Tuy nhi\u00EAn, ng\u01B0\u1EDDi d\u00E2n Philippines kh\u00F4ng quan t\u00E2m \u0111\u1EBFn tranh ch\u1EA5p Bi\u1EC3n \u0110\u00F4ng. \\n \"Asia Pulse\" c\u00F3 th\u1EC3 n\u00F3i l\u00E0 m\u1ED9t t\u1ED5 ch\u1EE9c b\u1ECF phi\u1EBFu, m\u1EE5c \u0111\u00EDch l\u00E0 t\u00ECm hi\u1EC3u mong mu\u1ED1n c\u1EE7a ng\u01B0\u1EDDi d\u00E2n Ch\u00E2u \u00C1. V\u00E0o th\u00E1ng 9 n\u0103m 2023, m\u1ED9t b\u00E1o c\u00E1o th\u0103m d\u00F2 \u00FD ki\u1EBFn \u200B\u200Bcho th\u1EA5y ch\u1EC9 c\u00F3 7% ng\u01B0\u1EDDi d\u00E2n Philippines quan t\u00E2m \u0111\u1EBFn tranh ch\u1EA5p Bi\u1EC3n \u0110\u00F4ng, trong khi 93% ng\u01B0\u1EDDi d\u00E2n th\u1EF1c s\u1EF1 ch\u1EC9 quan t\u00E2m \u0111\u1EBFn gi\u00E1 c\u1EA3 cao, l\u1EA1m ph\u00E1t, vi\u1EC7c l\u00E0m, v\u1EA5n \u0111\u1EC1 t\u1ED9i ph\u1EA1m gia t\u0103ng v\u00E0 v\u1EA5n \u0111\u1EC1 ma t\u00FAy, v.v. . . \\nGi\u1EDD \u0111\u00E2y, ngay c\u1EA3 gia \u0111\u00ECnh Marcos c\u0169ng b\u1ECB chia r\u1EBD. Em g\u00E1i c\u1EE7a Marcos, v\u1EDBi t\u01B0 c\u00E1ch l\u00E0 ch\u1EE7 t\u1ECBch \u1EE6y ban \u0110\u1ED1i ngo\u1EA1i Th\u01B0\u1EE3ng vi\u1EC7n Philippines, \u0111\u00E3 ph\u1EA3n \u0111\u1ED1i vi\u1EC7c Marcos quay sang Hoa K\u1EF3 v\u00E0 h\u00E0nh vi hung h\u0103ng c\u1EE7a Marcos \u1EDF ch\u00E2u \u00C1, \u0111\u1ED3ng th\u1EDDi ch\u1EC9 tr\u00EDch vi\u1EC7c tri\u1EC3n khai c\u00E1c c\u0103n c\u1EE9 c\u1EE7a M\u1EF9 \u1EDF Philippines v\u00E0 nh\u1EEFng c\u0103ng th\u1EB3ng do Hoa K\u1EF3 g\u00E2y ra \u1EDF Philippines. Bi\u1EC3n \u0110\u00F4ng. . \\n 02 \\n Tan Zhu: Sau khi th\u1ED5i ph\u1ED3ng s\u1EF1 c\u1ED1 \u0110\u00E1 Nh\u00E2n \u00C1i, kh\u00F4ng qu\u1ED1c gia n\u00E0o \u1EE7ng h\u1ED9 Philippines l\u00E0 qu\u1ED1c gia ASEAN. Ph\u1EA3i ch\u0103ng \u0111i\u1EC1u n\u00E0y c\u00F3 ngh\u0129a l\u00E0 Philippines \u0111\u00E3 b\u1ECB c\u00F4 l\u1EADp \u1EDF c\u1EA5p \u0111\u1ED9 ngo\u1EA1i giao khu v\u1EF1c do \u0111i \u0111\u1EA7u trong vi\u1EC7c g\u00E2y r\u1EAFc r\u1ED1i ? C\u00E1c n\u01B0\u1EDBc trong khu v\u1EF1c Bi\u1EC3n \u0110\u00F4ng mu\u1ED1n th\u1EA5y lo\u1EA1i Bi\u1EC3n \u0110\u00F4ng nh\u01B0 th\u1EBF n\u00E0o? \\n Herman Laurel: T\u00F4i ngh\u0129 ASEAN r\u00F5 r\u00E0ng kh\u00F4ng mu\u1ED1n c\u0103ng th\u1EB3ng leo thang. \\nTrong cu\u1ED9c ph\u1ECFng v\u1EA5n v\u1EDBi m\u1ED9t t\u1EDD b\u00E1o \u0111\u1ECBa ph\u01B0\u01A1ng \u1EDF Philippines, t\u00F4i n\u00F3i r\u1EB1ng Kh\u00F4ng qu\u00E2n Hoa K\u1EF3 v\u00E0 c\u00E1c l\u1EF1c l\u01B0\u1EE3ng qu\u00E2n s\u1EF1 kh\u00E1c \u0111\u00E3 can thi\u1EC7p v\u00E0o c\u00F4ng vi\u1EC7c c\u1EE7a Philippines \u1EDF Bi\u1EC3n \u0110\u00F4ng, v\u00E0 ASEAN kh\u00F4ng th\u00EDch s\u1EF1 can thi\u1EC7p c\u1EE7a M\u1EF9 v\u00E0o Bi\u1EC3n \u0110\u00F4ng. \\nM\u1EB7c d\u00F9 c\u00E1c n\u01B0\u1EDBc ASEAN r\u1EA5t l\u1ECBch s\u1EF1 v\u1EDBi nhau v\u00E0 kh\u00F4ng tr\u1EF1c ti\u1EBFp n\u00EAu t\u00EAn Philippines nh\u01B0ng h\u1ECD kh\u00F4ng tham gia c\u00E1c h\u00E0nh \u0111\u1ED9ng \u0111\u1ED9c l\u1EADp c\u1EE7a Philippines m\u00E0 c\u00F4 l\u1EADp Philippines. \\nT\u00F4i ngh\u0129 Th\u1EE7 t\u01B0\u1EDBng Singapore L\u00FD Hi\u1EC3n Long v\u00E0 c\u00E1c nh\u00E0 l\u00E3nh \u0111\u1EA1o kh\u00E1c \u0111\u00E3 \u0111\u01B0a ra m\u1ED9t s\u1ED1 b\u00ECnh lu\u1EADn gi\u00E1n ti\u1EBFp \u0111\u1EC1 c\u1EADp \u0111\u1EBFn Philippines v\u00E0 c\u1EA3nh b\u00E1o n\u01B0\u1EDBc n\u00E0y kh\u00F4ng n\u00EAn tr\u1EDF th\u00E0nh chi\u1EBFn tr\u01B0\u1EDDng. \u0110i\u1EC1u n\u00E0y th\u1EC3 hi\u1EC7n r\u1EA5t r\u00F5 m\u1ED1i quan ng\u1EA1i c\u1EE7a Singapore v\u00E0 truy\u1EC1n t\u1EA3i m\u1ED1i quan ng\u1EA1i t\u01B0\u01A1ng t\u1EF1 c\u1EE7a c\u00E1c n\u01B0\u1EDBc ASEAN. \\nT\u00F4i c\u0169ng \u0111ang li\u00EAn h\u1EC7 v\u1EDBi m\u1ED9t s\u1ED1 \u0111\u1EA1i s\u1EE9 qu\u00E1n ASEAN t\u1EA1i Manila. Trong cu\u1ED9c tr\u00F2 chuy\u1EC7n v\u1EDBi h\u1ECD, t\u00F4i \u0111\u00E3 nghe h\u1ECD b\u00E0y t\u1ECF m\u1ED1i quan ng\u1EA1i c\u1EE7a m\u1ED7i n\u01B0\u1EDBc v\u1EC1 c\u00E1c h\u00E0nh \u0111\u1ED9ng hi\u1EC7n t\u1EA1i c\u1EE7a Philippines \u1EDF Bi\u1EC3n \u0110\u00F4ng. \\nTr\u00EAn th\u1EF1c t\u1EBF, t\u00ECnh h\u00ECnh do Marcos v\u00E0 Hoa K\u1EF3 t\u1EA1o ra c\u0169ng \u0111\u00E3 khi\u1EBFn ng\u01B0\u1EDDi d\u00E2n c\u1EE7a ch\u00FAng t\u00F4i lo l\u1EAFng v\u1EC1 an ninh qu\u1ED1c gia. \\n Marcos v\u00E0 c\u00E1c th\u00E0nh vi\u00EAn n\u1ED9i c\u00E1c an ninh c\u1EE7a \u00F4ng \u0111\u00E3 th\u00F4ng b\u00E1o r\u1EB1ng h\u1ECD \u0111ang chuy\u1EC3n s\u1EF1 ch\u00FA \u00FD c\u1EE7a \u0111\u1EA5t n\u01B0\u1EDBc t\u1EEB an ninh n\u1ED9i b\u1ED9 sang an ninh b\u00EAn ngo\u00E0i, t\u1EE9c l\u00E0 h\u1ECD \u0111ang l\u00E0m \u0111i\u1EC1u \u0111\u00F3 m\u00E0 kh\u00F4ng th\u1EF1c s\u1EF1 t\u00ECm ra ai l\u00E0 m\u1ED1i \u0111e d\u1ECDa an ninh qu\u1ED1c gia, ho\u1EA1t \u0111\u1ED9ng \u0111\u1EC3 b\u1EA3o v\u1EC7 an ninh l\u00E3nh th\u1ED5. \\n S\u1EF1 thay \u0111\u1ED5i ch\u00EDnh s\u00E1ch n\u00E0y khi\u1EBFn m\u1ECDi ng\u01B0\u1EDDi l\u1EA7m t\u01B0\u1EDFng r\u1EB1ng ph\u1EA3i ch\u00FA \u00FD \u0111\u1EBFn an ninh b\u00EAn ngo\u00E0i, tuy nhi\u00EAn \u0111\u00E2y kh\u00F4ng ph\u1EA3i l\u00E0 m\u1ED1i \u0111e d\u1ECDa th\u1EF1c s\u1EF1 m\u00E0 ch\u1EC9 l\u00E0 m\u1ED9t \u1EA3o \u1EA3nh, khi\u1EBFn m\u1ECDi ng\u01B0\u1EDDi l\u01A1 l\u00E0 an ninh trong n\u01B0\u1EDBc v\u00E0 l\u00E0m t\u00ECnh h\u00ECnh an ninh trong n\u01B0\u1EDBc tr\u1EDF n\u00EAn t\u1ED3i t\u1EC7 h\u01A1n. \\n C\u00E1ch \u0111\u00E2y kh\u00F4ng l\u00E2u, m\u1ED9t v\u1EE5 \u0111\u00E1nh bom kh\u1EE7ng b\u1ED1 nghi\u00EAm tr\u1ECDng \u0111\u00E3 x\u1EA3y ra t\u1EA1i \u0110\u1EA1i h\u1ECDc Qu\u1ED1c gia Mindanao \u1EDF th\u00E0nh ph\u1ED1 Marawi, khi\u1EBFn nhi\u1EC1u ng\u01B0\u1EDDi Philippines thi\u1EC7t m\u1EA1ng; \u1EDF m\u1ED9t t\u1EC9nh kh\u00E1c, m\u1ED9t v\u1EE5 n\u1ED5 c\u0169ng x\u1EA3y ra tr\u00EAn m\u1ED9t chi\u1EBFc xe bu\u00FDt. \\n \\n \\n \\n \\n V\u00EC v\u1EADy, ch\u00EDnh ph\u1EE7 n\u00E0y th\u1EF1c s\u1EF1 \u0111ang g\u1EB7p r\u1EAFc r\u1ED1i li\u00EAn t\u1EE5c. Khi Marcos v\u1EA5p ph\u1EA3i s\u1EF1 ph\u1EA3n \u0111\u1ED1i \u0111\u1ED1i v\u1EDBi c\u00E1c ch\u00EDnh s\u00E1ch sai l\u1EA7m c\u1EE7a m\u00ECnh, s\u1EF1 \u1EE7ng h\u1ED9 c\u1EE7a c\u00F4ng ch\u00FAng \u0111\u1ED1i v\u1EDBi \u00F4ng \u0111\u00E3 gi\u1EA3m \u00EDt nh\u1EA5t 15%. \\n Nh\u1EEFng t\u00ECnh h\u00ECnh trong n\u01B0\u1EDBc n\u00E0y cho th\u1EA5y n\u0103m 2024 s\u1EBD l\u00E0 m\u1ED9t n\u0103m r\u1EA5t kh\u00F3 kh\u0103n \u0111\u1ED1i v\u1EDBi Marcos. \\nM\u1EF9 \u0111\u00E3 g\u00E2y \u00E1p l\u1EF1c l\u1EDBn, y\u00EAu c\u1EA7u Philippines ti\u1EBFp t\u1EE5c ki\u1EC1m ch\u1EBF Trung Qu\u1ED1c. Nh\u01B0ng t\u00F4i ngh\u0129 \u0111\u1EA1i \u0111a s\u1ED1 ng\u01B0\u1EDDi Philippines c\u0169ng \u0111ang l\u00E0m ph\u1EA7n vi\u1EC7c c\u1EE7a m\u00ECnh \u0111\u1EC3 c\u1ED1 g\u1EAFng t\u00E1i kh\u1EB3ng \u0111\u1ECBnh quan \u0111i\u1EC3m v\u00E0 quan \u0111i\u1EC3m th\u1EF1c s\u1EF1 c\u1EE7a ng\u01B0\u1EDDi Philippines v\u1EC1 Trung Qu\u1ED1c, m\u1ED9t n\u01B0\u1EDBc th\u00E2n thi\u1EC7n, \u1EA5m \u00E1p v\u00E0 hi\u1EC7u qu\u1EA3. \\n 03 \\n Zhu Tan: \u00D4ng t\u1EEBng ch\u1EC9 ra r\u1EB1ng qu\u00E1 tr\u00ECnh chuy\u1EC3n \u0111\u1ED5i kinh t\u1EBF v\u00E0 t\u0103ng tr\u01B0\u1EDFng b\u1EC1n v\u1EEFng c\u1EE7a Philippines kh\u00F4ng th\u1EC3 t\u00E1ch r\u1EDDi vi\u1EC7c duy tr\u00EC h\u1EE3p t\u00E1c ch\u1EB7t ch\u1EBD v\u1EDBi Trung Qu\u1ED1c v\u00E0 ASEAN. T\u1EA1i sao \u00F4ng l\u1EA1i \u0111\u01B0a ra nh\u1EADn \u0111\u1ECBnh nh\u01B0 v\u1EADy? \\n Herman Laurel: Sau khi Marcos nh\u1EADm ch\u1EE9c, ch\u00EDnh s\u00E1ch \u0111\u1ED1i ngo\u1EA1i \u0111\u1ED9c l\u1EADp v\u00E0 quan h\u1EC7 h\u1EEFu ngh\u1ECB c\u1EE7a ch\u00EDnh ph\u1EE7 Philippines v\u1EDBi Trung Qu\u1ED1c v\u00E0 c\u00E1c n\u01B0\u1EDBc l\u00E1ng gi\u1EC1ng ch\u00E2u \u00C1 kh\u00E1c \u0111\u00E3 quay \u0111\u1EA7u v\u00E0 quay sang Hoa K\u1EF3, \u0111i\u1EC1u n\u00E0y mang l\u1EA1i nhi\u1EC1u v\u1EA5n \u0111\u1EC1 cho Philippines, ti\u1EC1m n\u0103ng nghi\u00EAm tr\u1ECDng nh\u1EA5t t\u00E1c \u0111\u1ED9ng v\u1EABn c\u00F3 th\u1EC3 xu\u1EA5t hi\u1EC7n trong l\u0129nh v\u1EF1c kinh t\u1EBF. \\nR\u00F5 r\u00E0ng \u0111\u1ED9ng c\u01A1 ph\u1EE5c h\u1ED3i kinh t\u1EBF to\u00E0n c\u1EA7u sau cu\u1ED9c \u0110\u1EA1i suy tho\u00E1i k\u00E9o d\u00E0i hai n\u0103m r\u01B0\u1EE1i l\u00E0 Trung Qu\u1ED1c. \\ n Li\u00EAn minh Ch\u00E2u \u00C2u hi\u1EC7n \u0111ang trong v\u1EF1c th\u1EB3m suy tho\u00E1i. T\u1ED5ng s\u1EA3n ph\u1EA9m qu\u1ED1c n\u1ED9i (GDP) th\u1EF1c t\u1EBF c\u1EE7a Nh\u1EADt B\u1EA3n l\u1EA7n \u0111\u1EA7u ti\u00EAn t\u0103ng tr\u01B0\u1EDFng \u00E2m trong qu\u00FD 3 n\u0103m nay. Tuy nhi\u00EAn, Qu\u1EF9 Ti\u1EC1n t\u1EC7 Qu\u1ED1c t\u1EBF g\u1EA7n \u0111\u00E2y \u0111\u00E3 n\u00E2ng d\u1EF1 b\u00E1o t\u0103ng tr\u01B0\u1EDFng GDP c\u1EE7a Trung Qu\u1ED1c cho n\u0103m 2023 l\u00EAn 5,4% v\u00E0 n\u0103m 2024. T\u01B0\u01A1ng lai v\u1EABn t\u01B0\u01A1i s\u00E1ng. \\n B\u1EA1n ph\u1EA3i bi\u1EBFt r\u1EB1ng Trung Qu\u1ED1c kh\u00F4ng ch\u1EC9 l\u00E0 \u0111\u1ED1i t\u00E1c th\u01B0\u01A1ng m\u1EA1i l\u1EDBn nh\u1EA5t c\u1EE7a ASEAN m\u00E0 c\u00F2n l\u00E0 \u0111\u1ED1i t\u00E1c th\u01B0\u01A1ng m\u1EA1i l\u1EDBn nh\u1EA5t c\u1EE7a Philippines. \\n Tuy nhi\u00EAn, ch\u00EDnh quy\u1EC1n Marcos \u0111\u00E3 h\u1EE7y b\u1ECF nhi\u1EC1u d\u1EF1 \u00E1n x\u00E2y d\u1EF1ng c\u1EE7a Trung Qu\u1ED1c, m\u1EB7c d\u00F9 Trung Qu\u1ED1c kh\u00F4ng c\u00F3 b\u1EA5t k\u1EF3 t\u00E1c \u0111\u1ED9ng ti\u00EAu c\u1EF1c n\u00E0o v\u1EC1 kinh t\u1EBF \u0111\u1ED1i v\u1EDBi Philippines. \\n T\u00F4i \u0111\u00E3 nh\u1EAFc nh\u1EDF \u0111\u1ED3ng b\u00E0o m\u00ECnh r\u1EB1ng ch\u00FAng ta kh\u00F4ng \u0111\u01B0\u1EE3c qu\u00EAn b\u00E0i h\u1ECDc c\u1EE7a n\u0103m 2014. Khi \u0111\u00F3, d\u1EF1 \u0111o\u00E1n sai l\u1EA7m c\u1EE7a ch\u00EDnh ph\u1EE7 Aquino \u0111\u00E3 khi\u1EBFn s\u1ED1 chu\u1ED1i tr\u1ECB gi\u00E1 h\u00E0ng t\u1EF7 peso m\u00E0 n\u01B0\u1EDBc ta d\u1EF1 \u0111\u1ECBnh xu\u1EA5t sang Trung Qu\u1ED1c ban \u0111\u1EA7u kh\u00F4ng v\u00E0o \u0111\u01B0\u1EE3c Trung Qu\u1ED1c, c\u00E1c nh\u00E0 xu\u1EA5t kh\u1EA9u chu\u1ED1i Philippines ch\u1EC9 c\u00F3 th\u1EC3 \u0111\u1EE9ng nh\u00ECn chu\u1ED1i c\u1EE7a m\u00ECnh th\u1ED1i r\u1EEFa trong nh\u1EEFng l\u00F4 b\u1ECB t\u1EEB ch\u1ED1i trong h\u1ED3 s\u01A1. \\n V\u00EC v\u1EADy, c\u00F3 th\u1EC3 th\u1EA5y r\u1EB1ng m\u1ED9t d\u1EF1 \u0111o\u00E1n sai l\u1EA7m c\u1EE7a ch\u00EDnh ph\u1EE7 Philippines c\u00F3 th\u1EC3 d\u1EABn \u0111\u1EBFn t\u00ECnh th\u1EBF kh\u00F3 kh\u0103n m\u00E0 ng\u01B0\u1EDDi d\u00E2n Philippines b\u00ECnh th\u01B0\u1EDDng ng\u00E0y nay ph\u1EA3i \u0111\u1ED1i m\u1EB7t. Ch\u00FAng t\u00F4i li\u00EAn t\u1EE5c nh\u1EAFc nh\u1EDF ng\u01B0\u1EDDi d\u00E2n trong n\u01B0\u1EDBc r\u1EB1ng ch\u00FAng t\u00F4i kh\u00F4ng mu\u1ED1n nh\u1EEFng t\u00ECnh hu\u1ED1ng n\u00E0y x\u1EA3y ra l\u1EA7n n\u1EEFa. \\nC\u00F3 l\u1EBD ch\u1EC9 c\u00F3 Trung Qu\u1ED1c m\u1EDBi th\u1EF1c s\u1EF1 c\u00F3 th\u1EC3 th\u00FAc \u0111\u1EA9y s\u1EF1 ph\u00E1t tri\u1EC3n kinh t\u1EBF m\u00E0 Philippines c\u1EA7n trong nh\u1EEFng n\u0103m t\u1EDBi. L\u1EA5y n\u0103m t\u1EDBi l\u00E0m v\u00ED d\u1EE5, Hoa K\u1EF3 c\u00F3 th\u1EC3 kh\u00F4ng c\u00F3 \u0111\u1EE7 ngu\u1ED3n l\u1EF1c t\u00E0i ch\u00EDnh \u0111\u1EC3 h\u1ED7 tr\u1EE3 Philippines, Nh\u1EADt B\u1EA3n c\u00F3 th\u1EC3 kh\u00F4ng th\u1EC3 cung c\u1EA5p h\u1ED7 tr\u1EE3 do suy tho\u00E1i kinh t\u1EBF, Ng\u00E2n h\u00E0ng Ph\u00E1t tri\u1EC3n Ch\u00E2u \u00C1 v\u1EABn c\u00F2n m\u1ED9t s\u1ED1 d\u1EF1 \u00E1n c\u0169 c\u1EA7n \u0111\u01B0\u1EE3c ho\u00E0n thi\u1EC7n. ti\u1EBFp t\u1EE5c v\u00E0 c\u00F3 th\u1EC3 kh\u00F4ng c\u00F3 th\u1EDDi gian \u0111\u1EC3 quan t\u00E2m \u0111\u1EBFn Philippines. \\n Trong nh\u1EEFng th\u00E1ng t\u1EDBi, n\u1EBFu ch\u00FAng ta c\u00F3 th\u1EC3 ng\u0103n ch\u1EB7n th\u00E0nh c\u00F4ng vi\u1EC7c Hoa K\u1EF3 ph\u00E1 ho\u1EA1i m\u1ED1i quan h\u1EC7 t\u1ED1t \u0111\u1EB9p c\u1EE7a Philippines v\u1EDBi Trung Qu\u1ED1c, ch\u00FAng ta c\u00F3 th\u1EC3 ti\u1EBFp t\u1EE5c h\u01B0\u1EDBng t\u1EDBi m\u1ED9t t\u01B0\u01A1ng lai t\u1ED1t \u0111\u1EB9p h\u01A1n. \\n V\u00EC v\u1EADy, ch\u00FAng t\u00F4i hy v\u1ECDng r\u1EB1ng v\u00E0o n\u0103m 2024, Philippines c\u00F3 th\u1EC3 xoay chuy\u1EC3n t\u00ECnh th\u1EBF. \\n \\n \\n [Bi\u00EAn t\u1EADp vi\u00EAn: T\u1EC1 L\u00F4i]"
+#     message = "G\u1EA7n \u0111\u00E2y, Philippines \u0111\u00E3 nhi\u1EC1u l\u1EA7n g\u00E2y ra r\u1EAFc r\u1ED1i \u1EDF Bi\u1EC3n \u0110\u00F4ng, x\u00E2m nh\u1EADp \u0110\u00E1 Nh\u00E2n \u00C1i thu\u1ED9c qu\u1EA7n \u0111\u1EA3o Nam Sa c\u1EE7a Trung Qu\u1ED1c, \u0111\u1ED3ng th\u1EDDi ti\u1EBFp t\u1EE5c ph\u00F3ng \u0111\u1EA1i v\u00E0 c\u01B0\u1EDDng \u0111i\u1EC7u h\u00F3a, nh\u1EA7m l\u1EABn \u0111\u00FAng sai v\u00E0 \u0111\u01B0a ra nh\u1EEFng c\u00E1o bu\u1ED9c v\u00F4 l\u00FD ch\u1ED1ng l\u1EA1i Trung Qu\u1ED1c. \\n Ng\u01B0\u1EDDi Philippines ngh\u0129 g\u00EC v\u1EC1 h\u00E0nh \u0111\u1ED9ng c\u1EE7a ch\u00EDnh ph\u1EE7 Philippines? \\ n \u00D4ng Tan \u0111\u00E3 t\u00ECm ra Herman Laurel, m\u1ED9t nh\u00E0 b\u00ECnh lu\u1EADn ch\u00EDnh tr\u1ECB n\u1ED5i ti\u1EBFng \u1EDF Philippines, \u0111\u1ED3ng th\u1EDDi l\u00E0 gi\u00E1m \u0111\u1ED1c Vi\u1EC7n nghi\u00EAn c\u1EE9u chi\u1EBFn l\u01B0\u1EE3c \"Th\u1EBF k\u1EF7 ch\u00E2u \u00C1\" c\u1EE7a Philippines v\u00E0 l\u00E0 ng\u01B0\u1EDDi s\u00E1ng l\u1EADp Hi\u1EC7p h\u1ED9i nghi\u00EAn c\u1EE9u ch\u00EDnh s\u00E1ch BRICS c\u1EE7a Philippines. Herman Laurel, ng\u01B0\u1EDDi \u0111ang nghi\u00EAn c\u1EE9u v\u1EC1 ch\u00E2u \u00C1, \u0111\u1EB7c bi\u1EC7t l\u00E0 m\u1ED1i quan h\u1EC7 gi\u1EEFa Trung Qu\u1ED1c v\u00E0 ASEAN, chia s\u1EBB quan s\u00E1t c\u1EE7a m\u00ECnh. \\n 01 \\n Tan Zhu: K\u1EC3 t\u1EEB n\u1EEDa cu\u1ED1i n\u0103m nay, Philippines th\u01B0\u1EDDng xuy\u00EAn x\u00E2m chi\u1EBFm c\u00E1c \u0111\u1EA3o v\u00E0 b\u00E3i \u0111\u00E1 c\u1EE7a ch\u00FAng t\u00F4i \u1EDF Bi\u1EC3n \u0110\u00F4ng, l\u00E0m gia t\u0103ng c\u0103ng th\u1EB3ng \u1EDF Bi\u1EC3n \u0110\u00F4ng, \u0111\u1ED3ng th\u1EDDi vu kh\u1ED1ng Trung Qu\u1ED1c l\u00E0 k\u1EBB b\u1EAFt n\u1EA1t l\u1EDBn. , c\u00F3 nh\u1EEFng nh\u00F3m \u1EDF Philippines ph\u1EA3n \u0111\u1ED1i h\u00E0nh \u0111\u1ED9ng khi\u00EAu kh\u00EDch c\u1EE7a Philippines. B\u1EA1n c\u1EA3m th\u1EA5y th\u1EBF n\u00E0o v\u1EC1 \u0111i\u1EC1u n\u00E0y v\u1EDBi t\u01B0 c\u00E1ch l\u00E0 m\u1ED9t ng\u01B0\u1EDDi Philippines? Herman Laurel: T\u00F4i cho r\u1EB1ng C\u1EA3nh s\u00E1t bi\u1EC3n Philippines v\u00E0 m\u1ED9t s\u1ED1 t\u00E0u d\u00E2n s\u1EF1 Philippines \u0111\u00E3 c\u1ED1 t\u00ECnh g\u00E2y r\u1EAFc r\u1ED1i v\u00EC h\u1ECD \u0111ang th\u1EF1c hi\u1EC7n \u201CD\u1EF1 \u00E1n t\u1ED5ng th\u1EC3\u201D c\u1EE7a Hoa K\u1EF3. \\nM\u1EF9 khoe khoang d\u1EF1 \u00E1n n\u00E0y v\u1EDBi ch\u00FAng t\u00F4i n\u00EAn ch\u00FAng t\u00F4i bi\u1EBFt t\u00EAn d\u1EF1 \u00E1n. D\u1EF1 \u00E1n \u0111\u01B0\u1EE3c d\u1EABn d\u1EAFt b\u1EDFi \u0110\u1EA1i t\u00E1 Kh\u00F4ng qu\u00E2n Hoa K\u1EF3 \u0111\u00E3 ngh\u1EC9 h\u01B0u Raimundo Powell, l\u00E0m vi\u1EC7c v\u1EDBi c\u00E1c \u0111\u1ED1i t\u00E1c Philippines nh\u01B0 C\u1EA3nh s\u00E1t bi\u1EC3n Philippines v\u00E0 m\u1ED9t s\u1ED1 quan ch\u1EE9c c\u1EE7a B\u1ED9 Ngo\u1EA1i giao Philippines"
+#     qs = "Tóm tắt nội dung liên quan đến Biển Đông và Việt Nam"
+#     lg = "vi"
+#     ver = "en-llama"
+#     ans = summary_with_llama(qs, message, lg, version=ver, max_word_per_context=1024)
+#     print(ans)

function/tc_v2.py

@@ -0,0 +1,573 @@
+from tensorRT import inference
+import re
+from collections import Counter
+from vncorenlp import VnCoreNLP
+from nltk.tokenize import sent_tokenize
+import torch
+import datetime
+from sklearn.cluster import AgglomerativeClustering
+from sklearn.metrics.pairwise import cosine_similarity
+import numpy as np
+import json
+from . import utils
+import time
+from summary import text_summary, get_summary_bert
+from function.clean_text import normalize_text
+from .summary_with_llm import summary_with_llama
+from .translate import translate_text_multi_layer
+from scipy.spatial import distance
+import copy
+from .sentence_embbeding import embbeded_zh, embbeded_en, embedded_bge
+
+
+# from . import detect_time as dt
+
+device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+use_cuda = torch.cuda.is_available()
+print(torch.cuda.is_available())
+
+# annotator = VnCoreNLP('vncorenlp/VnCoreNLP-1.1.1.jar', port=9191, annotators="wseg,pos", max_heap_size='-Xmx2g')
+
+
+def detect_postaging(text_in):
+    word_segmented_text = annotator.annotate(text_in)
+    lst_k = []
+    for se in word_segmented_text["sentences"]:
+        for kw in se:
+            if kw["posTag"] in ("Np", "Ny", "N"):
+                if kw["posTag"] == "N" and "_" not in kw["form"]:
+                    continue
+                lst_k.append(kw["form"].replace("_", " "))
+    return list(set(lst_k))
+
+def clean_text(text_in):
+    doc = re.sub('<.*?>', '', text_in)
+    doc = re.sub('(function).*}', ' ', doc)
+    # link
+    doc = re.sub('(Nguồn)\s*?(http:\/\/).*?(\.htm)', ' ', doc)
+    doc = re.sub('(Nguồn)\s*?(http:\/\/).*?(\.html)', ' ', doc)
+    doc = re.sub('(Nguồn)\s*?(https:\/\/).*?(\/\/)', ' ', doc)
+    doc = re.sub('(Nguồn)\s*?(https:\/\/).*?(\.htm)', ' ', doc)
+    doc = re.sub('(Nguồn)\s*?(https:\/\/).*?(\.html)', ' ', doc)
+    doc = re.sub('(Nguồn)\s*?(https:\/\/).*?(\.vn)', ' ', doc)
+    doc = re.sub('(Nguồn)\s*?(https:\/\/).*?(\.net)', ' ', doc)
+    doc = re.sub('(Nguồn)\s*?(https:\/\/).*?(\.vgp)', ' ', doc)
+    doc = re.sub('(Nguồn)\s*?(http:\/\/).*?(\.vgp)', ' ', doc)
+
+    doc = re.sub('(http:\/\/).*?(\.htm)', ' ', doc)
+    doc = re.sub('(http:\/\/).*?(\.html)', ' ', doc)
+    doc = re.sub('(https:\/\/).*?(\/\/)', ' ', doc)
+    doc = re.sub('(https:\/\/).*?(\.htm)', ' ', doc)
+    doc = re.sub('(https:\/\/).*?(\.html)', ' ', doc)
+    doc = re.sub('(https:\/\/).*?(\.vn)', ' ', doc)
+    doc = re.sub('(https:\/\/).*?(\.net)', ' ', doc)
+    doc = re.sub('(https:\/\/).*?(\.vgp)', ' ', doc)
+    doc = re.sub('(http:\/\/).*?(\.vgp)', ' ', doc)
+    # escape sequence
+    doc = re.sub('\n', ' ', doc)
+    doc = re.sub('\t', ' ', doc)
+    doc = re.sub('\r', ' ', doc)
+
+    doc = normalize_text(doc)
+    return doc
+
+
+def data_cleaning(docs):
+    res = []
+    for d in docs:
+        if 'message' in d:
+            # css and js
+            doc = re.sub('<.*?>', '', d['message'])
+            doc = re.sub('(function).*}', ' ', doc)
+
+            # link
+            doc = re.sub('(Nguồn)\s*?(http:\/\/).*?(\.htm)', ' ', doc)
+            doc = re.sub('(Nguồn)\s*?(http:\/\/).*?(\.html)', ' ', doc)
+            doc = re.sub('(Nguồn)\s*?(https:\/\/).*?(\/\/)', ' ', doc)
+            doc = re.sub('(Nguồn)\s*?(https:\/\/).*?(\.htm)', ' ', doc)
+            doc = re.sub('(Nguồn)\s*?(https:\/\/).*?(\.html)', ' ', doc)
+            doc = re.sub('(Nguồn)\s*?(https:\/\/).*?(\.vn)', ' ', doc)
+            doc = re.sub('(Nguồn)\s*?(https:\/\/).*?(\.net)', ' ', doc)
+            doc = re.sub('(Nguồn)\s*?(https:\/\/).*?(\.vgp)', ' ', doc)
+            doc = re.sub('(Nguồn)\s*?(http:\/\/).*?(\.vgp)', ' ', doc)
+
+            doc = re.sub('(http:\/\/).*?(\.htm)', ' ', doc)
+            doc = re.sub('(http:\/\/).*?(\.html)', ' ', doc)
+            doc = re.sub('(https:\/\/).*?(\/\/)', ' ', doc)
+            doc = re.sub('(https:\/\/).*?(\.htm)', ' ', doc)
+            doc = re.sub('(https:\/\/).*?(\.html)', ' ', doc)
+            doc = re.sub('(https:\/\/).*?(\.vn)', ' ', doc)
+            doc = re.sub('(https:\/\/).*?(\.net)', ' ', doc)
+            doc = re.sub('(https:\/\/).*?(\.vgp)', ' ', doc)
+            doc = re.sub('(http:\/\/).*?(\.vgp)', ' ', doc)
+            # escape sequence
+            doc = re.sub('\n', ' ', doc)
+            doc = re.sub('\t', ' ', doc)
+            doc = re.sub('\r', ' ', doc)
+
+            d['message'] = doc
+            res.append(d)
+    return res
+
+
+def segment(docs, lang="vi"):
+    segmented_docs = []
+    for d in docs:
+        # print(d)
+        # if len(d.get('message', "")) > 8000 or len(d.get('message', "")) < 100:
+        if len(d.get('message', "")) > 8000:
+            continue
+        if 'snippet' not in d:
+            continue
+        try:
+            if lang == "vi":
+                snippet = d.get('snippet', "")
+                segmented_snippet = ""
+                segmented_sentences_snippet = annotator.tokenize(snippet)
+                for sentence in segmented_sentences_snippet:
+                    segmented_snippet += ' ' + ' '.join(sentence)
+                segmented_snippet = segmented_snippet.replace('\xa0', '')
+                d['segmented_snippet'] = segmented_snippet
+            segmented_docs.append(d)
+        except Exception:
+            pass
+    return segmented_docs
+
+
+def timestamp_to_date(timestamp):
+    return datetime.datetime.fromtimestamp(timestamp).strftime('%d/%m/%Y')
+
+
+def re_ranking(result_topic, vectors_prompt, sorted_field):
+    lst_score = []
+    lst_ids = []
+    lst_top = []
+    try:
+        for k in result_topic:
+            lst_ids.append(k)
+            if  not sorted_field.strip():
+                    lst_top.append(len(result_topic[k]))
+            else:
+                lst_top.append(result_topic[k][0]['max_score'])
+            vector_center = result_topic[k][0]["vector"]
+            max_score = 11.0
+            for vec in vectors_prompt:
+                score = distance.cosine(np.array(vec), np.array(vector_center))
+                if score < max_score:
+                    max_score = score
+            lst_score.append(max_score)
+            result_topic[k][0]["similarity_score"] = max_score
+            for d in result_topic[k]:
+                d["similarity_score"] = max_score
+            del result_topic[k][0]["vector"]
+        idx = np.argsort(np.array(lst_score))
+    except Exception as ve:
+        return [], lst_ids, lst_top
+    return idx, lst_ids, lst_top
+
+def post_processing(response, top_cluster=5, top_sentence=5, topn_summary=5, sorted_field='', max_doc_per_cluster = 50, delete_message=True, prompt="", hash_str: str= "", vectors_prompt: list = []):
+    print(f'[INFO] sorted_field: {sorted_field}')
+    MAX_DOC_PER_CLUSTER = max_doc_per_cluster
+
+    lst_ids = []
+    lst_top = []
+    lst_res = []
+    idx = []
+    if prompt:
+        idx, lst_ids, lst_top = re_ranking(response, vectors_prompt, sorted_field)
+        print("idx_prompt: ", idx)
+    if len(prompt) == 0 or len(idx) == 0:
+        for i in response:
+            lst_ids.append(i)
+            if  not sorted_field.strip():
+                lst_top.append(len(response[i]))
+            else:
+                lst_top.append(response[i][0]['max_score'])
+        idx = np.argsort(np.array(lst_top))[::-1] 
+        print("idx_not_prompt: ", idx)
+    if top_cluster == -1:
+        top_cluster = len(idx)
+    for i in idx[: top_cluster]:
+        ik = lst_ids[i]
+        if top_sentence == -1:
+            top_sentence = len(response[ik])
+        lst_check_title = []
+        lst_check_not_title = []
+        i_c_t = 0
+        for resss in response[ik]:
+            r_title = resss.get("title", "")
+            if r_title and not r_title.endswith("..."):
+                lst_check_title.append(resss)
+                i_c_t += 1
+            else:
+                lst_check_not_title.append(resss)
+            if i_c_t == top_sentence:
+                break
+        if i_c_t == top_sentence:
+            lst_res.append(lst_check_title)
+        else:
+            lst_check_title.extend(lst_check_not_title)
+            lst_res.append(lst_check_title[:top_sentence])
+        #lst_res.append(response[ik][:top_sentence])
+    dict_res = {}
+    for i in range(len(lst_res)):
+        dict_res[str(i + 1)] = lst_res[i][:MAX_DOC_PER_CLUSTER]
+        for j in range(min(len(dict_res[str(i + 1)]), 3)):
+            dict_res[str(i + 1)][0]["title_summarize"].append(dict_res[str(i + 1)][j].get("snippet", ""))
+        # t11 = time.time()
+        summary_text = get_summary_bert(dict_res[str(i + 1)][0].get("message", ""), dict_res[str(i + 1)][0].get("lang", "vi"), topn=topn_summary, title=dict_res[str(i + 1)][0].get("title", ""), snippet=dict_res[str(i + 1)][0].get("snippet", ""))
+        # print("time_summary: ", time.time() - t11)
+        if len(summary_text) < 10:
+            summary_text = dict_res[str(i + 1)][0].get("snippet", "")
+            if len(summary_text) < 10:
+                summary_text = dict_res[str(i + 1)][0].get("title", "")
+        summary_text = utils.remove_image_keyword(summary_text)
+        # if prompt:
+        #     if dict_res[str(i + 1)][0].get("message", ""):
+        #         src_lang = dict_res[str(i + 1)][0].get("lang", "")
+        #         print("src_lang: ", src_lang)
+        #         print("summary_text: ", summary_text)
+        #         summary_text = translate_text_multi_layer(src_lang, "vi", summary_text)
+        #         text_tran = translate_text_multi_layer(src_lang, "vi", dict_res[str(i + 1)][0].get("message", ""))
+        #         ans_from_llama = summary_with_llama(prompt, text_tran, "vi", version="vi-llama", max_word_per_context=1000)
+        #         print("ans_from_llama: ", ans_from_llama)
+        #     summary_text = summary_text + "$$$$\n" + ans_from_llama
+        #     print("summary_text: ", summary_text, len(summary_text))
+        dict_res[str(i + 1)][0]["content_summary"] = summary_text
+        dict_res[str(i + 1)][0]["num_of_post"] = len(lst_res[i])
+        kew_phares = []
+        dict_res[str(i + 1)][0]["topic_keywords"] = kew_phares
+        
+        # print("delete_message: ", delete_message)
+        if delete_message:
+            for j in range(len(dict_res[str(i + 1)])):
+                if "message" in dict_res[str(i + 1)][j]:
+                    del dict_res[str(i + 1)][j]["message"]
+    
+    with open(f"log_llm/topic_result_after_postprocessing/{hash_str}.json", "w") as f:
+        dict_log_pos = {}
+        for k in dict_res:
+            dict_log_pos[k] = copy.deepcopy(dict_res[k])
+            for d in dict_log_pos[k]:
+                if "message" in d:
+                    del d["message"]
+                if "vector" in d:
+                    del d["vector"]
+        json.dump(dict_log_pos, f, ensure_ascii= False)
+    return dict_res
+
+
+def get_lang(docs):
+    lang_vi = 0
+    lang_en = 0
+    dict_lang = {}
+    for d in docs:
+        lang = d.get("lang", "")
+        if lang not in dict_lang:
+            dict_lang[lang] = 0
+        dict_lang[lang] += 1
+        # if d.get("lang", "") == "vi":
+        #     lang_vi += 1
+        # else:
+        #     lang_en += 1
+    lst_lang = []
+    lst_cnt = []
+    for k in dict_lang:
+        lst_lang.append(k)
+        lst_cnt.append(dict_lang[k])
+    idx_max = np.argsort(np.array(lst_cnt))[::-1][0]
+    lang = lst_lang[int(idx_max)]
+    
+    if lang.startswith("zh_"):
+        lang = "zh"
+    print("lang: ", lang, lst_cnt[int(idx_max)])
+    return lang
+
+
+def topic_clustering(docs, distance_threshold, top_cluster=5, top_sentence=5, topn_summary=5, sorted_field='', max_doc_per_cluster=50, 
+                        delete_message=True, prompt="", type_cluster:str = "single", hash_str: str= "", id_topic=""):
+    # global model, model_en
+    with open("data/topic_name.txt") as f:
+        dict_topic_name = json.load(f)
+    topic_name_relevant = dict_topic_name.get(id_topic , "")
+    docs = docs[:30000]
+    lang = get_lang(docs)
+    if type_cluster == "complete" and lang == "zh":
+        distance_threshold = 0.4
+    if type_cluster == "complete" and lang == "en":
+        distance_threshold = 0.4
+        # type_cluster = "single"
+
+    result = {}
+    cluster_score = {}
+    cluster_real_vectors = {}
+    # docs = segment(docs, lang=lang)
+
+    t1 = time.time()
+    if len(docs) < 1:
+        return result
+    elif len(docs) == 1:
+        return {
+            "0": docs
+        }
+    vec_prompt = []
+    prompt_strips = []
+    # prompt = ""
+    if topic_name_relevant:
+        prompt_split = topic_name_relevant.split("#####")
+        for prom in prompt_split:
+            sys_p = prom.strip().split("$$$$")
+            if len(sys_p) == 1:
+                prompt_strips.append(prom.strip())
+            else:
+                prompt_strips.append(sys_p[1].strip())
+        if lang == "zh":
+            vec_prompt = embbeded_zh(prompt_split)
+        elif lang == "en":
+            vec_prompt = embbeded_en(prompt_split)
+        else:
+            vec_prompt = inference.encode(prompt_split, lang=lang)
+    if lang == "zh":
+        features = [doc.get('title', "") + ". " + doc.get('snippet', "") for doc in docs]
+        vectors = embbeded_zh(features)
+        # vectors = embedded_bge(features)
+        if len(vectors) == 0:
+            print(f"[WARNING] Embedded {lang}: {len(vectors)} / {len(features)}")
+            vectors = inference.encode(features, lang=lang)
+        # vectors = model.encode(features, show_progress_bar=False)
+    elif lang == "en":
+        features = [doc.get('title', "") + ". " + doc.get('snippet', "") for doc in docs]
+        vectors = embbeded_en(features)
+        # vectors = embedded_bge(features)
+        if len(vectors) == 0:
+            print(f"[WARNING] Embedded {lang}: {len(vectors)} / {len(features)}")
+            vectors = inference.encode(features, lang=lang)
+    else:
+        features = [doc.get('title', "") + ". " + doc.get('snippet', "") for doc in docs]
+        # vectors = embedded_bge(features)
+        # if len(vectors) == 0:
+        #     print(f"[WARNING] Embedded {lang}: {len(vectors)} / {len(features)}")
+        vectors = inference.encode(features, lang=lang)
+        # vectors = model_en.encode(features, show_progress_bar=False)
+    clusteror = AgglomerativeClustering(n_clusters=None, compute_full_tree=True, affinity='cosine',
+                                        linkage=type_cluster, distance_threshold=distance_threshold)
+    clusteror.fit(vectors)
+    matrix_vec = np.stack(vectors, axis=0)
+    print(f"Time encode + clustering: {time.time() - t1} {clusteror.n_clusters_}")
+    for i in range(clusteror.n_clusters_):
+        result[str(i + 1)] = []
+        cluster_score[str(i + 1)] = 0
+        ids = clusteror.labels_ # == i
+        # cluster_real_vectors[str(i + 1)] = re_clustering(ids, matrix_vec, distance_threshold, max_doc_per_cluster)
+
+    for i in range(len(clusteror.labels_)):
+        cluster_no = clusteror.labels_[i]
+        # if any((cluster_real_vectors[str(cluster_no+1)][:] == vectors[i]).all(1)):
+        if docs[i].get('domain','') not in ["cungcau.vn","baomoi.com","news.skydoor.net"]:
+            response_doc = {}
+            response_doc = docs[i]
+            score = response_doc.get('score', 0)
+            if not docs[i].get('message','').strip():
+                continue 
+            if score > cluster_score[str(cluster_no + 1)]:
+                cluster_score[str(cluster_no + 1)] = score
+            if 'domain' in docs[i]:
+                response_doc['domain'] = docs[i]['domain']
+            if 'url' in docs[i]:
+                response_doc['url'] = docs[i]['url']
+            if 'title' in docs[i]:
+                response_doc['title'] = clean_text(docs[i]['title'])
+            if 'snippet' in docs[i]:
+                response_doc['snippet'] = clean_text(docs[i]['snippet'])
+            if 'created_time' in docs[i]:
+                response_doc['created_time'] = docs[i]['created_time']
+            if "sentiment" in docs[i]:
+                response_doc['sentiment'] = docs[i]['sentiment']
+            if 'message' in docs[i]:
+                title = docs[i].get('title','')
+                snippet = docs[i].get('snippet','')
+                message = docs[i].get('message','')
+                if title.strip():
+                    split_mess = message.split(title)
+                    if len(split_mess) > 1:
+                        message = title.join(split_mess[1:])
+                if snippet.strip():
+                    split_mess = message.split(snippet)
+                    if len(split_mess) > 1:
+                        message = snippet.join(split_mess[1:])
+
+                response_doc['message'] = clean_text(message)
+            if 'id' in docs[i]:
+                response_doc['id'] = docs[i]['id']
+            # response_doc['score'] = 0.0
+            response_doc['title_summarize'] = []
+            response_doc['content_summary'] = ""
+            response_doc['total_facebook_viral'] = 0
+            response_doc["vector"] = np.array(vectors[i]).tolist()
+            result[str(cluster_no + 1)].append(response_doc)
+    empty_clus_ids = []
+    for x in result:
+        result[x] = sorted(result[x], key=lambda i: -len(i.get('message','')))
+        if len( result[x]) > 0:
+            if len(result[x]) > 1:
+                result[x] = check_duplicate_title_domain(result[x])
+            result[x][0]['num_docs'] = len(result[x])
+            result[x][0]['max_score'] = cluster_score[x]
+        else:
+            empty_clus_ids.append(x)
+    
+    for x in empty_clus_ids:
+        result.pop(x,None)
+    # result = dict(sorted(result.items(), key=lambda i: -len(i[1])))[:top_cluster]
+    with open(f"log_llm/topic_result_before_postprocessing/{hash_str}.json", "w") as f:
+        dict_log = {}
+        for k in result:
+            dict_log[k] = copy.deepcopy(result[k])
+            for d in dict_log[k]:
+                if "message" in d:
+                    del d["message"]
+                if "vector" in d:
+                    del d["vector"]
+        json.dump(dict_log, f, ensure_ascii= False)
+    return post_processing(result, top_cluster=top_cluster, top_sentence=top_sentence, topn_summary=topn_summary, sorted_field = sorted_field, max_doc_per_cluster=max_doc_per_cluster, delete_message=delete_message,
+                            prompt=topic_name_relevant, hash_str=hash_str, vectors_prompt=vec_prompt)
+
+def check_duplicate_title_domain(docs):
+    lst_title_domain = [f"{d.get('domain', '')} {d.get('title','')}" for d in docs]
+    for i in range(1,len(lst_title_domain) -1):
+        for j in range(i+1,len(lst_title_domain)):
+            if lst_title_domain[j] == lst_title_domain[i]:
+                lst_title_domain[j] = 'dup'
+    lst_filter_docs = [docs[i] for i,x in enumerate(lst_title_domain) if x != 'dup']
+    return lst_filter_docs
+def convert_date(text):
+    text = text.replace(".", "/")
+    text = text.replace("-", "/")
+    return text
+
+
+def check_keyword(sentence):
+    keyword = ['sáng', 'trưa', 'chiều', 'tối', 'đến', 'hôm', 'ngày', 'tới']
+    for k in keyword:
+        if k in sentence:
+            return True
+    return False
+
+
+def extract_events_and_time(docs, publish_date):
+    def standardize(date_str):
+        return date_str.replace('.', '/').replace('-', '/')
+
+    def add_0(date_str):
+
+        date_str = date_str.split('/')
+        res = []
+        for o in date_str:
+            o = re.sub('\s+', '', o)
+            if len(o) < 2:
+                o = '0' + o
+            res.append(o)
+        date_str = '/'.join(res)
+        return date_str
+
+    def get_date_list(reg, sentence):
+        find_object = re.finditer(reg, sentence)
+        date_list = [x.group() for x in find_object]
+        return date_list
+
+    year = publish_date.split('/')[2]
+
+    # dd/mm/yyyy
+    reg_exp_1 = '(\D|^)(?:0?[1-9]|[12][0-9]|3[01])[- \/.](?:0?[1-9]|1[012])[- \/.]([12]([0-9]){3})(\D|$)'
+    # #mm/yyyy
+    # reg_exp_5 = '(\D|^)(?:0?[1-9]|1[012])[- \/.]([12]([0-9]){3})(\D|$)'
+    # dd/mm
+    reg_exp_2 = '(\D|^)(?:0?[1-9]|[12][0-9]|3[01])[- \/.](?:0?[1-9]|1[012])(\D|$)'
+
+    # ngày  dd tháng mm năm yyyy
+    reg_exp_3 = '(ngày)\s*\d{1,2}\s*(tháng)\s*\d{1,2}\s*(năm)\s*\d{4}'
+    # ngày dd tháng mm
+    reg_exp_4 = '(ngày)\s*\d{1,2}\s*(tháng)\s*\d{1,2}'
+
+    result = []
+    for d in docs:
+        text = d['message']
+        for sentence in sent_tokenize(text):
+            lower_sentence = sentence.lower()
+            c = re.search(reg_exp_3, sentence.lower())
+            d = re.search(reg_exp_4, sentence.lower())
+            # e = re.search(reg_exp_5, sentence.lower())
+            a = re.search(reg_exp_1, sentence)
+            b = re.search(reg_exp_2, sentence)
+            #
+            if (a or b or c or d) and check_keyword(lower_sentence):
+                date_list = get_date_list(reg_exp_1, lower_sentence)
+                date_entity = ''
+                if date_list:
+                    date_entity = add_0(standardize(date_list[0]))
+                elif get_date_list(reg_exp_2, lower_sentence):
+                    date_list = get_date_list(reg_exp_2, lower_sentence)
+                    date_entity = add_0(standardize(date_list[0]) + '/' + year)
+                elif get_date_list(reg_exp_3, lower_sentence):
+                    date_list = get_date_list(reg_exp_3, lower_sentence)
+
+                    date_entity = date_list[0].replace('ngày', '').replace('tháng', '').replace('năm', '').strip()
+                    date_entity = re.sub('\s+', ' ', date_entity)
+                    date_entity = date_entity.replace(' ', '/')
+                    date_entity = add_0(date_entity)
+                else:
+                    date_list = get_date_list(reg_exp_4, lower_sentence)
+                    if date_list != []:
+                        date_entity = date_list[0].replace('ngày', '').replace('tháng', '').replace('năm', '').strip()
+                        date_entity = re.sub('\s+', ' ', date_entity)
+                        date_entity = date_entity.replace(' ', '/')
+                        date_entity = date_entity + '/' + year
+                        date_entity = add_0(date_entity)
+                result.append((sentence, date_entity))
+    return result
+
+def find_index_nearest_vector(cluster, vectors):
+    # Compute the centroid of the cluster
+    centroid = np.mean(cluster, axis=0, keepdims=True)
+    
+    # Calculate the Euclidean distance between each vector and the centroid
+    distances = cosine_similarity(centroid, vectors)
+    
+    # Find the index of the vector with the minimum distance
+    nearest_index = np.argmin(distances, axis=1)
+    
+    
+    return nearest_index
+
+def re_clustering(ids, vectors, distance_threshold, max_doc_per_cluster):
+    sub_vectors = vectors[ids]
+
+    try:
+        if sub_vectors.shape[0] < 2:
+            return sub_vectors
+        sub_clusteror = AgglomerativeClustering(n_clusters=None, compute_full_tree=True, affinity='cosine',
+                        linkage='complete', distance_threshold=0.12)
+        sub_clusteror.fit(sub_vectors)
+        dict_cluster = {id_clus: sub_vectors[sub_clusteror.labels_ == id_clus] for id_clus in range(sub_clusteror.n_clusters_)}
+        dict_num_vec = {id_clus: v.shape[0] for id_clus, v in dict_cluster.items()}
+
+        max_num_cluster = max(dict_num_vec, key=dict_num_vec.get)
+        other_vectors = sub_vectors[sub_clusteror.labels_ != max_num_cluster]
+
+        # if other_vectors.shape[0]:
+        #     while dict_num_vec[max_num_cluster] < max_doc_per_cluster:
+        #         tmp_index_vec = find_index_nearest_vector(dict_cluster[max_num_cluster], other_vectors)
+        #         dict_cluster[max_num_cluster] = np.vstack((dict_cluster[max_num_cluster], other_vectors[tmp_index_vec]))
+        #         dict_num_vec[max_num_cluster] += 1
+        #         if other_vectors.shape[0] != 1:
+        #             other_vectors = np.delete(other_vectors, tmp_index_vec, axis=0)
+        #         else:
+        #             break
+        cosine_scores = cosine_similarity(dict_cluster[max_num_cluster], dict_cluster[max_num_cluster])
+        with open("/home/vietle/topic-clustering/log_score.txt", "a") as f:
+            f.write(str(cosine_scores) + "\n")
+        return dict_cluster[max_num_cluster]
+    except Exception as e:
+        with open("/home/vietle/topic-clustering/log_clustering_diemtin/log_cluster_second.txt", "a") as f:
+            f.write(str(e)+"$$"+json.dumps({"ids": ids.tolist(), "vectors": vectors.tolist()}))
+        return sub_vectors 

function/topic_clustering.py

@@ -0,0 +1,458 @@
+from email import message
+import re
+from vncorenlp import VnCoreNLP
+from nltk.tokenize import sent_tokenize
+import torch
+from sentence_transformers import SentenceTransformer
+import datetime
+from sklearn.cluster import AgglomerativeClustering
+from sklearn.metrics.pairwise import cosine_similarity
+import numpy as np
+import requests
+import json
+from . import utils
+import time
+from summary import text_summary, get_summary_bert
+from function.clean_text import normalize_text
+# from . import detect_time as dt
+
+device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+model = SentenceTransformer('model/distiluse-base-multilingual-cased-v2').to(device)
+# model = SentenceTransformer('VoVanPhuc/sup-SimCSE-VietNamese-phobert-base').to(device)
+# model.save('model/distiluse-base-multilingual-cased-v2')
+
+use_cuda = torch.cuda.is_available()
+print(torch.cuda.is_available())
+if torch.cuda.is_available():
+    model_en = SentenceTransformer('model/paraphrase-mpnet-base-v2').to(device)
+else:
+    model_en = model
+# model_en.save('model/paraphrase-mpnet-base-v2')
+annotator = VnCoreNLP('vncorenlp/VnCoreNLP-1.1.1.jar', port=9191, annotators="wseg,pos", max_heap_size='-Xmx8g')
+
+
+def detect_postaging(text_in):
+    word_segmented_text = annotator.annotate(text_in)
+    lst_k = []
+    for se in word_segmented_text["sentences"]:
+        for kw in se:
+            if kw["posTag"] in ("Np", "Ny", "N"):
+                if kw["posTag"] == "N" and "_" not in kw["form"]:
+                    continue
+                lst_k.append(kw["form"].replace("_", " "))
+    return list(set(lst_k))
+
+def clean_text(text_in):
+    doc = re.sub('<.*?>', '', text_in)
+    doc = re.sub('(function).*}', ' ', doc)
+    # link
+    doc = re.sub('(Nguồn)\s*?(http:\/\/).*?(\.htm)', ' ', doc)
+    doc = re.sub('(Nguồn)\s*?(http:\/\/).*?(\.html)', ' ', doc)
+    doc = re.sub('(Nguồn)\s*?(https:\/\/).*?(\/\/)', ' ', doc)
+    doc = re.sub('(Nguồn)\s*?(https:\/\/).*?(\.htm)', ' ', doc)
+    doc = re.sub('(Nguồn)\s*?(https:\/\/).*?(\.html)', ' ', doc)
+    doc = re.sub('(Nguồn)\s*?(https:\/\/).*?(\.vn)', ' ', doc)
+    doc = re.sub('(Nguồn)\s*?(https:\/\/).*?(\.net)', ' ', doc)
+    doc = re.sub('(Nguồn)\s*?(https:\/\/).*?(\.vgp)', ' ', doc)
+    doc = re.sub('(Nguồn)\s*?(http:\/\/).*?(\.vgp)', ' ', doc)
+
+    doc = re.sub('(http:\/\/).*?(\.htm)', ' ', doc)
+    doc = re.sub('(http:\/\/).*?(\.html)', ' ', doc)
+    doc = re.sub('(https:\/\/).*?(\/\/)', ' ', doc)
+    doc = re.sub('(https:\/\/).*?(\.htm)', ' ', doc)
+    doc = re.sub('(https:\/\/).*?(\.html)', ' ', doc)
+    doc = re.sub('(https:\/\/).*?(\.vn)', ' ', doc)
+    doc = re.sub('(https:\/\/).*?(\.net)', ' ', doc)
+    doc = re.sub('(https:\/\/).*?(\.vgp)', ' ', doc)
+    doc = re.sub('(http:\/\/).*?(\.vgp)', ' ', doc)
+    # escape sequence
+    doc = re.sub('\n', ' ', doc)
+    doc = re.sub('\t', ' ', doc)
+    doc = re.sub('\r', ' ', doc)
+
+    doc = normalize_text(doc)
+    return doc
+
+
+def data_cleaning(docs):
+    res = []
+    for d in docs:
+        if 'message' in d:
+            # css and js
+            doc = re.sub('<.*?>', '', d['message'])
+            doc = re.sub('(function).*}', ' ', doc)
+
+            # link
+            doc = re.sub('(Nguồn)\s*?(http:\/\/).*?(\.htm)', ' ', doc)
+            doc = re.sub('(Nguồn)\s*?(http:\/\/).*?(\.html)', ' ', doc)
+            doc = re.sub('(Nguồn)\s*?(https:\/\/).*?(\/\/)', ' ', doc)
+            doc = re.sub('(Nguồn)\s*?(https:\/\/).*?(\.htm)', ' ', doc)
+            doc = re.sub('(Nguồn)\s*?(https:\/\/).*?(\.html)', ' ', doc)
+            doc = re.sub('(Nguồn)\s*?(https:\/\/).*?(\.vn)', ' ', doc)
+            doc = re.sub('(Nguồn)\s*?(https:\/\/).*?(\.net)', ' ', doc)
+            doc = re.sub('(Nguồn)\s*?(https:\/\/).*?(\.vgp)', ' ', doc)
+            doc = re.sub('(Nguồn)\s*?(http:\/\/).*?(\.vgp)', ' ', doc)
+
+            doc = re.sub('(http:\/\/).*?(\.htm)', ' ', doc)
+            doc = re.sub('(http:\/\/).*?(\.html)', ' ', doc)
+            doc = re.sub('(https:\/\/).*?(\/\/)', ' ', doc)
+            doc = re.sub('(https:\/\/).*?(\.htm)', ' ', doc)
+            doc = re.sub('(https:\/\/).*?(\.html)', ' ', doc)
+            doc = re.sub('(https:\/\/).*?(\.vn)', ' ', doc)
+            doc = re.sub('(https:\/\/).*?(\.net)', ' ', doc)
+            doc = re.sub('(https:\/\/).*?(\.vgp)', ' ', doc)
+            doc = re.sub('(http:\/\/).*?(\.vgp)', ' ', doc)
+            # escape sequence
+            doc = re.sub('\n', ' ', doc)
+            doc = re.sub('\t', ' ', doc)
+            doc = re.sub('\r', ' ', doc)
+
+            d['message'] = doc
+            res.append(d)
+    return res
+
+
+def segment(docs, lang="vi"):
+    segmented_docs = []
+    for d in docs:
+        # if len(d.get('message', "")) > 8000 or len(d.get('message', "")) < 100:
+        if len(d.get('message', "")) > 8000:
+            continue
+        if 'snippet' not in d:
+            continue
+        try:
+            if lang == "vi":
+                snippet = d.get('snippet', "")
+                segmented_snippet = ""
+                segmented_sentences_snippet = annotator.tokenize(snippet)
+                for sentence in segmented_sentences_snippet:
+                    segmented_snippet += ' ' + ' '.join(sentence)
+                segmented_snippet = segmented_snippet.replace('\xa0', '')
+                d['segmented_snippet'] = segmented_snippet
+            segmented_docs.append(d)
+        except Exception:
+            pass
+    return segmented_docs
+
+
+def timestamp_to_date(timestamp):
+    return datetime.datetime.fromtimestamp(timestamp).strftime('%d/%m/%Y')
+
+
+def post_processing(response, top_cluster=5, top_sentence=5, topn_summary=5, sorted_field='', max_doc_per_cluster = 50):
+    print(f'[INFO] sorted_field: {sorted_field}')
+    MAX_DOC_PER_CLUSTER = max_doc_per_cluster
+
+    lst_ids = []
+    lst_top = []
+    lst_res = []
+    for i in response:
+        lst_ids.append(i)
+
+        if  not sorted_field.strip():
+            lst_top.append(len(response[i]))
+        else:
+            lst_top.append(response[i][0]['max_score'])
+        
+    idx = np.argsort(np.array(lst_top))[::-1] 
+    if top_cluster == -1:
+        top_cluster = len(idx)
+    for i in idx[: top_cluster]:
+        ik = lst_ids[i]
+        if top_sentence == -1:
+            top_sentence = len(response[ik])
+        lst_check_title = []
+        lst_check_not_title = []
+        i_c_t = 0
+        for resss in response[ik]:
+            r_title = resss.get("title", "")
+            if r_title and not r_title.endswith("..."):
+                lst_check_title.append(resss)
+                i_c_t += 1
+            else:
+                lst_check_not_title.append(resss)
+            if i_c_t == top_sentence:
+                break
+        if i_c_t == top_sentence:
+            lst_res.append(lst_check_title)
+        else:
+            lst_check_title.extend(lst_check_not_title)
+            lst_res.append(lst_check_title[:top_sentence])
+        #lst_res.append(response[ik][:top_sentence])
+    dict_res = {}
+    for i in range(len(lst_res)):
+        dict_res[str(i + 1)] = lst_res[i][:MAX_DOC_PER_CLUSTER]
+        for j in range(min(len(dict_res[str(i + 1)]), 3)):
+            dict_res[str(i + 1)][0]["title_summarize"].append(dict_res[str(i + 1)][j].get("snippet", ""))
+        summary_text = get_summary_bert(dict_res[str(i + 1)][0].get("message", ""), lang=dict_res[str(i + 1)][0].get("lang", "vi"), topn=topn_summary, title=dict_res[str(i + 1)][0].get("title", ""), snippet=dict_res[str(i + 1)][0].get("snippet", ""))
+        if len(summary_text) < 10:
+            summary_text = dict_res[str(i + 1)][0].get("snippet", "")
+            if len(summary_text) < 10:
+                summary_text = dict_res[str(i + 1)][0].get("title", "")
+        dict_res[str(i + 1)][0]["content_summary"] = utils.remove_image_keyword(summary_text)
+        kew_phares = []
+        dict_res[str(i + 1)][0]["topic_keywords"] = kew_phares
+        for j in range(len(dict_res[str(i + 1)])):
+            if "message" in dict_res[str(i + 1)][j]:
+                del dict_res[str(i + 1)][j]["message"]
+    return dict_res
+
+
+def get_lang(docs):
+    lang_vi = 0
+    lang_en = 0
+    for d in docs:
+        if d.get("lang", "") == "vi":
+            lang_vi += 1
+        else:
+            lang_en += 1
+    if lang_vi >= lang_en:
+        return "vi"
+    return "en"
+
+
+# def topic_clustering(docs, distance_threshold, top_cluster=5, top_sentence=5, topn_summary=5, sorted_field=''):
+#     global model, model_en
+#     docs = docs[:30000]
+#     lang = get_lang(docs)
+#     result = {}
+#     docs = segment(docs, lang=lang)
+#     if len(docs) < 2:
+#         return result
+#     if lang == "vi":
+#         features = [doc.get('title', "") + ". " + doc.get('snippet', "") for doc in docs]
+#         vectors = model.encode(features, show_progress_bar=False)
+#     else:
+#         features = [doc.get('title', "") + ". " + doc.get('snippet', "") for doc in docs]
+#         vectors = model_en.encode(features, show_progress_bar=False)
+#     clusteror = AgglomerativeClustering(n_clusters=None, compute_full_tree=True, affinity='cosine',
+#                                         linkage='single', distance_threshold=distance_threshold)
+#     clusteror.fit(vectors)
+#     print(clusteror.n_clusters_)
+#     for i in range(clusteror.n_clusters_):
+#         result[str(i + 1)] = []
+#     for i in range(len(clusteror.labels_)):
+#         cluster_no = clusteror.labels_[i]   
+#         if docs[i].get('domain','') not in ["cungcau.vn","baomoi.com","news.skydoor.net"]:
+#             response_doc = {}
+#             response_doc = docs[i]
+#             if 'domain' in docs[i]:
+#                 response_doc['domain'] = docs[i]['domain']
+#             if 'url' in docs[i]:
+#                 response_doc['url'] = docs[i]['url']
+#             if 'title' in docs[i]:
+#                 response_doc['title'] = clean_text(docs[i]['title'])
+#             if 'snippet' in docs[i]:
+#                 response_doc['snippet'] = clean_text(docs[i]['snippet'])
+#             if 'created_time' in docs[i]:
+#                 response_doc['created_time'] = docs[i]['created_time']
+#             if 'message' in docs[i]:
+#                 title = docs[i].get('title','')
+#                 snippet = docs[i].get('snippet','')
+#                 message = docs[i].get('message','')
+#                 if title.strip():
+#                     split_mess = message.split(title)
+#                     if len(split_mess) > 1:
+#                         message = title.join(split_mess[1:])
+#                 if snippet.strip():
+#                     split_mess = message.split(snippet)
+#                     if len(split_mess) > 1:
+#                         message = snippet.join(split_mess[1:])
+
+#                 response_doc['message'] = clean_text(message)
+#             if 'id' in docs[i]:
+#                 response_doc['id'] = docs[i]['id']
+#             response_doc['score'] = 0.0
+#             response_doc['title_summarize'] = []
+#             response_doc['content_summary'] = ""
+#             response_doc['total_facebook_viral'] = 0
+#             result[str(cluster_no + 1)].append(response_doc)
+    
+#     empty_clus_ids = []
+#     for x in result:
+#         result[x] = sorted(result[x], key=lambda i: -len(i.get('message','')))
+#         if len( result[x]) > 0:
+#             if len(result[x]) > 1:
+#                 result[x] = check_duplicate_title_domain(result[x])
+#             result[x][0]['num_docs'] = len(result[x])
+#         else:
+#             empty_clus_ids.append(x)
+    
+#     for x in empty_clus_ids:
+#         result.pop(x,None)
+#     # result = dict(sorted(result.items(), key=lambda i: -len(i[1])))[:top_cluster]
+#     return post_processing(result, top_cluster=top_cluster, top_sentence=top_sentence, topn_summary=topn_summary, sorted_field = sorted_field)
+
+def topic_clustering(docs, distance_threshold, top_cluster=5, top_sentence=5, topn_summary=5, sorted_field='', max_doc_per_cluster=50, delete_message=True):
+    global model, model_en
+    docs = docs[:30000]
+    lang = get_lang(docs)
+    result = {}
+    cluster_score = {}
+    # docs = segment(docs, lang=lang)
+    if len(docs) < 2:
+        return result
+    if lang == "vi":
+        features = [doc.get('title', "") + ". " + doc.get('snippet', "") for doc in docs]
+        vectors = model.encode(features, show_progress_bar=False)
+    else:
+        features = [doc.get('title', "") + ". " + doc.get('snippet', "") for doc in docs]
+        vectors = model_en.encode(features, show_progress_bar=False)
+    clusteror = AgglomerativeClustering(n_clusters=None, compute_full_tree=True, affinity='cosine',
+                                        linkage='single', distance_threshold=distance_threshold)
+    clusteror.fit(vectors)
+    print(clusteror.n_clusters_)
+    for i in range(clusteror.n_clusters_):
+        result[str(i + 1)] = []
+        cluster_score[str(i + 1)] = 0
+    for i in range(len(clusteror.labels_)):
+        cluster_no = clusteror.labels_[i]   
+        if docs[i].get('domain','') not in ["cungcau.vn","baomoi.com","news.skydoor.net"]:
+            response_doc = {}
+            response_doc = docs[i]
+            score = response_doc.get('score', 0)
+            if not docs[i].get('message','').strip():
+                continue 
+            if score > cluster_score[str(cluster_no + 1)]:
+                cluster_score[str(cluster_no + 1)] = score
+            if 'domain' in docs[i]:
+                response_doc['domain'] = docs[i]['domain']
+            if 'url' in docs[i]:
+                response_doc['url'] = docs[i]['url']
+            if 'title' in docs[i]:
+                response_doc['title'] = clean_text(docs[i]['title'])
+            if 'snippet' in docs[i]:
+                response_doc['snippet'] = clean_text(docs[i]['snippet'])
+            if 'created_time' in docs[i]:
+                response_doc['created_time'] = docs[i]['created_time']
+            if 'message' in docs[i]:
+                title = docs[i].get('title','')
+                snippet = docs[i].get('snippet','')
+                message = docs[i].get('message','')
+                if title.strip():
+                    split_mess = message.split(title)
+                    if len(split_mess) > 1:
+                        message = title.join(split_mess[1:])
+                if snippet.strip():
+                    split_mess = message.split(snippet)
+                    if len(split_mess) > 1:
+                        message = snippet.join(split_mess[1:])
+
+                response_doc['message'] = clean_text(message)
+            if 'id' in docs[i]:
+                response_doc['id'] = docs[i]['id']
+            # response_doc['score'] = 0.0
+            response_doc['title_summarize'] = []
+            response_doc['content_summary'] = ""
+            response_doc['total_facebook_viral'] = 0
+            result[str(cluster_no + 1)].append(response_doc)
+    
+    empty_clus_ids = []
+    for x in result:
+        result[x] = sorted(result[x], key=lambda i: -len(i.get('message','')))
+        if len( result[x]) > 0:
+            if len(result[x]) > 1:
+                result[x] = check_duplicate_title_domain(result[x])
+            result[x][0]['num_docs'] = len(result[x])
+            result[x][0]['max_score'] = cluster_score[x]
+        else:
+            empty_clus_ids.append(x)
+    
+    for x in empty_clus_ids:
+        result.pop(x,None)
+    # result = dict(sorted(result.items(), key=lambda i: -len(i[1])))[:top_cluster]
+    return post_processing(result, top_cluster=top_cluster, top_sentence=top_sentence, topn_summary=topn_summary, sorted_field = sorted_field, max_doc_per_cluster=max_doc_per_cluster)
+
+def check_duplicate_title_domain(docs):
+    lst_title_domain = [f"{d.get('domain', '')} {d.get('title','')}" for d in docs]
+    for i in range(1,len(lst_title_domain) -1):
+        for j in range(i+1,len(lst_title_domain)):
+            if lst_title_domain[j] == lst_title_domain[i]:
+                lst_title_domain[j] = 'dup'
+    lst_filter_docs = [docs[i] for i,x in enumerate(lst_title_domain) if x != 'dup']
+    return lst_filter_docs
+def convert_date(text):
+    text = text.replace(".", "/")
+    text = text.replace("-", "/")
+    return text
+
+
+def check_keyword(sentence):
+    keyword = ['sáng', 'trưa', 'chiều', 'tối', 'đến', 'hôm', 'ngày', 'tới']
+    for k in keyword:
+        if k in sentence:
+            return True
+    return False
+
+
+def extract_events_and_time(docs, publish_date):
+    def standardize(date_str):
+        return date_str.replace('.', '/').replace('-', '/')
+
+    def add_0(date_str):
+
+        date_str = date_str.split('/')
+        res = []
+        for o in date_str:
+            o = re.sub('\s+', '', o)
+            if len(o) < 2:
+                o = '0' + o
+            res.append(o)
+        date_str = '/'.join(res)
+        return date_str
+
+    def get_date_list(reg, sentence):
+        find_object = re.finditer(reg, sentence)
+        date_list = [x.group() for x in find_object]
+        return date_list
+
+    year = publish_date.split('/')[2]
+
+    # dd/mm/yyyy
+    reg_exp_1 = '(\D|^)(?:0?[1-9]|[12][0-9]|3[01])[- \/.](?:0?[1-9]|1[012])[- \/.]([12]([0-9]){3})(\D|$)'
+    # #mm/yyyy
+    # reg_exp_5 = '(\D|^)(?:0?[1-9]|1[012])[- \/.]([12]([0-9]){3})(\D|$)'
+    # dd/mm
+    reg_exp_2 = '(\D|^)(?:0?[1-9]|[12][0-9]|3[01])[- \/.](?:0?[1-9]|1[012])(\D|$)'
+
+    # ngày  dd tháng mm năm yyyy
+    reg_exp_3 = '(ngày)\s*\d{1,2}\s*(tháng)\s*\d{1,2}\s*(năm)\s*\d{4}'
+    # ngày dd tháng mm
+    reg_exp_4 = '(ngày)\s*\d{1,2}\s*(tháng)\s*\d{1,2}'
+
+    result = []
+    for d in docs:
+        text = d['message']
+        for sentence in sent_tokenize(text):
+            lower_sentence = sentence.lower()
+            c = re.search(reg_exp_3, sentence.lower())
+            d = re.search(reg_exp_4, sentence.lower())
+            # e = re.search(reg_exp_5, sentence.lower())
+            a = re.search(reg_exp_1, sentence)
+            b = re.search(reg_exp_2, sentence)
+            #
+            if (a or b or c or d) and check_keyword(lower_sentence):
+                date_list = get_date_list(reg_exp_1, lower_sentence)
+                date_entity = ''
+                if date_list:
+                    date_entity = add_0(standardize(date_list[0]))
+                elif get_date_list(reg_exp_2, lower_sentence):
+                    date_list = get_date_list(reg_exp_2, lower_sentence)
+                    date_entity = add_0(standardize(date_list[0]) + '/' + year)
+                elif get_date_list(reg_exp_3, lower_sentence):
+                    date_list = get_date_list(reg_exp_3, lower_sentence)
+
+                    date_entity = date_list[0].replace('ngày', '').replace('tháng', '').replace('năm', '').strip()
+                    date_entity = re.sub('\s+', ' ', date_entity)
+                    date_entity = date_entity.replace(' ', '/')
+                    date_entity = add_0(date_entity)
+                else:
+                    date_list = get_date_list(reg_exp_4, lower_sentence)
+                    if date_list != []:
+                        date_entity = date_list[0].replace('ngày', '').replace('tháng', '').replace('năm', '').strip()
+                        date_entity = re.sub('\s+', ' ', date_entity)
+                        date_entity = date_entity.replace(' ', '/')
+                        date_entity = date_entity + '/' + year
+                        date_entity = add_0(date_entity)
+                result.append((sentence, date_entity))
+    return result

function/topic_clustering_mnews.py

@@ -0,0 +1,339 @@
+import re
+from vncorenlp import VnCoreNLP
+from nltk.tokenize import sent_tokenize
+import torch
+from sentence_transformers import SentenceTransformer
+import datetime
+from sklearn.cluster import AgglomerativeClustering
+
+import numpy as np
+import requests
+import json
+from . import utils
+import time
+from summary import text_summary, get_summary_bert
+# from . import detect_time as dt
+
+device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+model = SentenceTransformer('VoVanPhuc/sup-SimCSE-VietNamese-phobert-base').to(device)
+model_en = SentenceTransformer('paraphrase-mpnet-base-v2').to(device)
+annotator = VnCoreNLP('vncorenlp/VnCoreNLP-1.1.1.jar', port=9191, annotators="wseg,pos", max_heap_size='-Xmx8g')
+
+
+def detect_postaging(text_in):
+    word_segmented_text = annotator.annotate(text_in)
+    lst_k = []
+    for se in word_segmented_text["sentences"]:
+        for kw in se:
+            if kw["posTag"] in ("Np", "Ny", "N"):
+                if kw["posTag"] == "N" and "_" not in kw["form"]:
+                    continue
+                lst_k.append(kw["form"].replace("_", " "))
+    return list(set(lst_k))
+
+def clean_text(text_in):
+    doc = re.sub('<.*?>', '', text_in)
+    doc = re.sub('(function).*}', ' ', doc)
+    # link
+    doc = re.sub('(Nguồn)\s*?(http:\/\/).*?(\.htm)', ' ', doc)
+    doc = re.sub('(Nguồn)\s*?(http:\/\/).*?(\.html)', ' ', doc)
+    doc = re.sub('(Nguồn)\s*?(https:\/\/).*?(\/\/)', ' ', doc)
+    doc = re.sub('(Nguồn)\s*?(https:\/\/).*?(\.htm)', ' ', doc)
+    doc = re.sub('(Nguồn)\s*?(https:\/\/).*?(\.html)', ' ', doc)
+    doc = re.sub('(Nguồn)\s*?(https:\/\/).*?(\.vn)', ' ', doc)
+    doc = re.sub('(Nguồn)\s*?(https:\/\/).*?(\.net)', ' ', doc)
+    doc = re.sub('(Nguồn)\s*?(https:\/\/).*?(\.vgp)', ' ', doc)
+    doc = re.sub('(Nguồn)\s*?(http:\/\/).*?(\.vgp)', ' ', doc)
+
+    doc = re.sub('(http:\/\/).*?(\.htm)', ' ', doc)
+    doc = re.sub('(http:\/\/).*?(\.html)', ' ', doc)
+    doc = re.sub('(https:\/\/).*?(\/\/)', ' ', doc)
+    doc = re.sub('(https:\/\/).*?(\.htm)', ' ', doc)
+    doc = re.sub('(https:\/\/).*?(\.html)', ' ', doc)
+    doc = re.sub('(https:\/\/).*?(\.vn)', ' ', doc)
+    doc = re.sub('(https:\/\/).*?(\.net)', ' ', doc)
+    doc = re.sub('(https:\/\/).*?(\.vgp)', ' ', doc)
+    doc = re.sub('(http:\/\/).*?(\.vgp)', ' ', doc)
+    # escape sequence
+    doc = re.sub('\n', ' ', doc)
+    doc = re.sub('\t', ' ', doc)
+    doc = re.sub('\r', ' ', doc)
+    return doc
+
+
+def data_cleaning(docs):
+    res = []
+    for d in docs:
+        if 'message' in d:
+            # css and js
+            doc = re.sub('<.*?>', '', d['message'])
+            doc = re.sub('(function).*}', ' ', doc)
+
+            # link
+            doc = re.sub('(Nguồn)\s*?(http:\/\/).*?(\.htm)', ' ', doc)
+            doc = re.sub('(Nguồn)\s*?(http:\/\/).*?(\.html)', ' ', doc)
+            doc = re.sub('(Nguồn)\s*?(https:\/\/).*?(\/\/)', ' ', doc)
+            doc = re.sub('(Nguồn)\s*?(https:\/\/).*?(\.htm)', ' ', doc)
+            doc = re.sub('(Nguồn)\s*?(https:\/\/).*?(\.html)', ' ', doc)
+            doc = re.sub('(Nguồn)\s*?(https:\/\/).*?(\.vn)', ' ', doc)
+            doc = re.sub('(Nguồn)\s*?(https:\/\/).*?(\.net)', ' ', doc)
+            doc = re.sub('(Nguồn)\s*?(https:\/\/).*?(\.vgp)', ' ', doc)
+            doc = re.sub('(Nguồn)\s*?(http:\/\/).*?(\.vgp)', ' ', doc)
+
+            doc = re.sub('(http:\/\/).*?(\.htm)', ' ', doc)
+            doc = re.sub('(http:\/\/).*?(\.html)', ' ', doc)
+            doc = re.sub('(https:\/\/).*?(\/\/)', ' ', doc)
+            doc = re.sub('(https:\/\/).*?(\.htm)', ' ', doc)
+            doc = re.sub('(https:\/\/).*?(\.html)', ' ', doc)
+            doc = re.sub('(https:\/\/).*?(\.vn)', ' ', doc)
+            doc = re.sub('(https:\/\/).*?(\.net)', ' ', doc)
+            doc = re.sub('(https:\/\/).*?(\.vgp)', ' ', doc)
+            doc = re.sub('(http:\/\/).*?(\.vgp)', ' ', doc)
+            # escape sequence
+            doc = re.sub('\n', ' ', doc)
+            doc = re.sub('\t', ' ', doc)
+            doc = re.sub('\r', ' ', doc)
+
+            d['message'] = doc
+            res.append(d)
+    return res
+
+
+def segment(docs, lang="vi"):
+    segmented_docs = []
+    for d in docs:
+        # if len(d.get('message', "")) > 8000 or len(d.get('message', "")) < 100:
+        #     continue
+        if 'snippet' not in d and 'title' not in d:
+            continue
+        try:
+            if lang == "vi":
+                snippet = d.get('snippet', "")
+                segmented_snippet = ""
+                segmented_sentences_snippet = annotator.tokenize(snippet)
+                for sentence in segmented_sentences_snippet:
+                    segmented_snippet += ' ' + ' '.join(sentence)
+                segmented_snippet = segmented_snippet.replace('\xa0', '')
+                d['segmented_snippet'] = segmented_snippet
+            segmented_docs.append(d)
+        except Exception:
+            pass
+    return segmented_docs
+
+
+def timestamp_to_date(timestamp):
+    return datetime.datetime.fromtimestamp(timestamp).strftime('%d/%m/%Y')
+
+
+def sort_content(lst_res):
+    lst_content = []
+    lst_cnt = []
+    for i in range(len(lst_res)):
+        lst_cnt.append(len(lst_res[i].get("message", "")))
+    id_sort = np.argsort(np.array(lst_cnt))[::-1]
+    for i in id_sort:
+        lst_content.append(lst_res[i])
+    return lst_content
+
+
+
+def post_processing(response, top_cluster=5, top_sentence=5, topn_summary=5):
+    lst_ids = []
+    lst_top = []
+    lst_res = []
+    for i in response:
+        lst_ids.append(i)
+        lst_top.append(len(response[i]))
+    idx = np.argsort(np.array(lst_top))[::-1]
+    if top_cluster == -1:
+        top_cluster = len(idx)
+    for i in idx[: top_cluster]:
+        ik = lst_ids[i]
+        if top_sentence == -1:
+            top_sentence = len(response[ik])
+        lst_check_title = []
+        lst_check_not_title = []
+        i_c_t = 0
+        response_sort = sort_content(response[ik].copy())
+        for resss in response_sort:
+            if resss.get("title", ""):
+                lst_check_title.append(resss)
+                i_c_t += 1
+            else:
+                lst_check_not_title.append(resss)
+            if i_c_t == top_sentence:
+                break
+        if i_c_t == top_sentence:
+            lst_res.append(lst_check_title)
+        else:
+            lst_check_title.extend(lst_check_not_title)
+            lst_res.append(lst_check_title[:top_sentence])
+    dict_res = {}
+    for i in range(len(lst_res)):
+        dict_res[str(i + 1)] = lst_res[i]
+        for j in range(min(len(dict_res[str(i + 1)]), 3)):
+            dict_res[str(i + 1)][0]["title_summarize"].append(dict_res[str(i + 1)][j].get("snippet", ""))
+        summary_text = get_summary_bert(dict_res[str(i + 1)][0].get("message", ""), lang = dict_res[str(i + 1)][0].get("lang", "vi"), topn=topn_summary)
+        if len(summary_text) < 10:
+            summary_text = dict_res[str(i + 1)][0].get("snippet", "")
+            if len(summary_text) < 10:
+                summary_text = dict_res[str(i + 1)][0].get("title", "")
+        dict_res[str(i + 1)][0]["content_summary"] = utils.remove_image_keyword(summary_text)
+        kew_phares = []
+        dict_res[str(i + 1)][0]["topic_keywords"] = kew_phares
+        for j in range(len(dict_res[str(i + 1)])):
+            if "message" in dict_res[str(i + 1)][j]:
+                del dict_res[str(i + 1)][j]["message"]
+    return dict_res
+
+
+def get_lang(docs):
+    lang_vi = 0
+    lang_en = 0
+    docs_lang_vi = []
+    docs_lang_en = []
+    for d in docs:
+        if d.get("lang", "") == "en":
+            lang_en += 1
+            docs_lang_en.append(d)
+        else:
+            lang_vi += 1
+            docs_lang_vi.append(d)
+    if lang_vi > lang_en:
+        return "vi", docs_lang_vi
+    return "en", docs_lang_en
+
+
+def topic_clustering(docs, distance_threshold, top_cluster=5, top_sentence=5, topn_summary=5, benchmark_id=1):
+    global model, model_en
+    lang, docs = get_lang(docs)
+    result = {}
+    docs = segment(docs, lang=lang)
+    print("docs segment: ", len(docs))
+    if len(docs) < 2:
+        return result
+    if lang == "vi":
+        features = [doc.get('title', "") + ". " + doc.get('snippet', "") for doc in docs]
+        vectors = model.encode(features, show_progress_bar=False)
+    else:
+        features = [doc.get('title', "") + ". " + doc.get('snippet', "") for doc in docs]
+        vectors = model_en.encode(features, show_progress_bar=False)
+    clusteror = AgglomerativeClustering(n_clusters=None, compute_full_tree=True, affinity='cosine',
+                                        linkage='single', distance_threshold=distance_threshold)
+    clusteror.fit(vectors)
+    print(clusteror.n_clusters_)
+    for i in range(clusteror.n_clusters_):
+        result[str(i + 1)] = []
+    for i in range(len(clusteror.labels_)):
+        cluster_no = clusteror.labels_[i]
+        response_doc = {}
+        if 'url' in docs[i]:
+            response_doc['url'] = docs[i]['url']
+        if 'domain' in docs[i]:
+            response_doc['domain'] = docs[i]['domain']
+        if 'title' in docs[i]:
+            response_doc['title'] = clean_text(docs[i]['title'])
+        if 'snippet' in docs[i]:
+            response_doc['snippet'] = clean_text(docs[i]['snippet'])
+        if 'created_time' in docs[i]:
+            response_doc['created_time'] = docs[i]['created_time']
+        if 'message' in docs[i]:
+            response_doc['message'] = clean_text(docs[i]['message'])
+        if 'id' in docs[i]:
+            response_doc['id'] = docs[i]['id']
+        response_doc['score'] = 0.0
+        response_doc['title_summarize'] = []
+        response_doc['content_summary'] = ""
+        response_doc['total_facebook_viral'] = 0
+        result[str(cluster_no + 1)].append(response_doc)
+    # print("before filter: ", len(result))
+    # result = smart_filter(result, benchmark_id=benchmark_id)
+    # print("after filter: ", len(result))
+    return post_processing(result, top_cluster=top_cluster, top_sentence=top_sentence, topn_summary=topn_summary)
+
+
+def convert_date(text):
+    text = text.replace(".", "/")
+    text = text.replace("-", "/")
+    return text
+
+
+def check_keyword(sentence):
+    keyword = ['sáng', 'trưa', 'chiều', 'tối', 'đến', 'hôm', 'ngày', 'tới']
+    for k in keyword:
+        if k in sentence:
+            return True
+    return False
+
+
+def extract_events_and_time(docs, publish_date):
+    def standardize(date_str):
+        return date_str.replace('.', '/').replace('-', '/')
+
+    def add_0(date_str):
+
+        date_str = date_str.split('/')
+        res = []
+        for o in date_str:
+            o = re.sub('\s+', '', o)
+            if len(o) < 2:
+                o = '0' + o
+            res.append(o)
+        date_str = '/'.join(res)
+        return date_str
+
+    def get_date_list(reg, sentence):
+        find_object = re.finditer(reg, sentence)
+        date_list = [x.group() for x in find_object]
+        return date_list
+
+    year = publish_date.split('/')[2]
+
+    # dd/mm/yyyy
+    reg_exp_1 = '(\D|^)(?:0?[1-9]|[12][0-9]|3[01])[- \/.](?:0?[1-9]|1[012])[- \/.]([12]([0-9]){3})(\D|$)'
+    # #mm/yyyy
+    # reg_exp_5 = '(\D|^)(?:0?[1-9]|1[012])[- \/.]([12]([0-9]){3})(\D|$)'
+    # dd/mm
+    reg_exp_2 = '(\D|^)(?:0?[1-9]|[12][0-9]|3[01])[- \/.](?:0?[1-9]|1[012])(\D|$)'
+
+    # ngày  dd tháng mm năm yyyy
+    reg_exp_3 = '(ngày)\s*\d{1,2}\s*(tháng)\s*\d{1,2}\s*(năm)\s*\d{4}'
+    # ngày dd tháng mm
+    reg_exp_4 = '(ngày)\s*\d{1,2}\s*(tháng)\s*\d{1,2}'
+
+    result = []
+    for d in docs:
+        text = d['message']
+        for sentence in sent_tokenize(text):
+            lower_sentence = sentence.lower()
+            c = re.search(reg_exp_3, sentence.lower())
+            d = re.search(reg_exp_4, sentence.lower())
+            # e = re.search(reg_exp_5, sentence.lower())
+            a = re.search(reg_exp_1, sentence)
+            b = re.search(reg_exp_2, sentence)
+            #
+            if (a or b or c or d) and check_keyword(lower_sentence):
+                date_list = get_date_list(reg_exp_1, lower_sentence)
+                date_entity = ''
+                if date_list:
+                    date_entity = add_0(standardize(date_list[0]))
+                elif get_date_list(reg_exp_2, lower_sentence):
+                    date_list = get_date_list(reg_exp_2, lower_sentence)
+                    date_entity = add_0(standardize(date_list[0]) + '/' + year)
+                elif get_date_list(reg_exp_3, lower_sentence):
+                    date_list = get_date_list(reg_exp_3, lower_sentence)
+
+                    date_entity = date_list[0].replace('ngày', '').replace('tháng', '').replace('năm', '').strip()
+                    date_entity = re.sub('\s+', ' ', date_entity)
+                    date_entity = date_entity.replace(' ', '/')
+                    date_entity = add_0(date_entity)
+                else:
+                    date_list = get_date_list(reg_exp_4, lower_sentence)
+                    if date_list != []:
+                        date_entity = date_list[0].replace('ngày', '').replace('tháng', '').replace('năm', '').strip()
+                        date_entity = re.sub('\s+', ' ', date_entity)
+                        date_entity = date_entity.replace(' ', '/')
+                        date_entity = date_entity + '/' + year
+                        date_entity = add_0(date_entity)
+                result.append((sentence, date_entity))
+    return result

function/topic_clustering_not_summary.py

@@ -0,0 +1,463 @@
+from email import message
+import re
+from turtle import title
+from vncorenlp import VnCoreNLP
+from nltk.tokenize import sent_tokenize
+import torch
+from sentence_transformers import SentenceTransformer
+import datetime
+from sklearn.cluster import AgglomerativeClustering
+from sklearn.metrics.pairwise import cosine_similarity
+import numpy as np
+import requests
+import json
+from . import utils
+import time
+from summary import text_summary, get_summary_bert
+from function.clean_text import normalize_text
+# from . import detect_time as dt
+
+device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+# model = SentenceTransformer('model/distiluse-base-multilingual-cased-v2').to(device)
+model = SentenceTransformer('model/paraphrase-multilingual-MiniLM-L12-v2')
+
+# model = SentenceTransformer('VoVanPhuc/sup-SimCSE-VietNamese-phobert-base').to(device)
+# model.save('model/distiluse-base-multilingual-cased-v2')
+
+use_cuda = torch.cuda.is_available()
+print(torch.cuda.is_available())
+if torch.cuda.is_available():
+    model_en = SentenceTransformer('model/paraphrase-mpnet-base-v2').to(device)
+else:
+    model_en = model
+# model_en.save('model/paraphrase-mpnet-base-v2')
+annotator = VnCoreNLP('vncorenlp/VnCoreNLP-1.1.1.jar', port=9191, annotators="wseg,pos", max_heap_size='-Xmx8g')
+
+
+def detect_postaging(text_in):
+    word_segmented_text = annotator.annotate(text_in)
+    lst_k = []
+    for se in word_segmented_text["sentences"]:
+        for kw in se:
+            if kw["posTag"] in ("Np", "Ny", "N"):
+                if kw["posTag"] == "N" and "_" not in kw["form"]:
+                    continue
+                lst_k.append(kw["form"].replace("_", " "))
+    return list(set(lst_k))
+
+def clean_text(text_in):
+    doc = re.sub('<.*?>', '', text_in)
+    doc = re.sub('(function).*}', ' ', doc)
+    # link
+    doc = re.sub('(Nguồn)\s*?(http:\/\/).*?(\.htm)', ' ', doc)
+    doc = re.sub('(Nguồn)\s*?(http:\/\/).*?(\.html)', ' ', doc)
+    doc = re.sub('(Nguồn)\s*?(https:\/\/).*?(\/\/)', ' ', doc)
+    doc = re.sub('(Nguồn)\s*?(https:\/\/).*?(\.htm)', ' ', doc)
+    doc = re.sub('(Nguồn)\s*?(https:\/\/).*?(\.html)', ' ', doc)
+    doc = re.sub('(Nguồn)\s*?(https:\/\/).*?(\.vn)', ' ', doc)
+    doc = re.sub('(Nguồn)\s*?(https:\/\/).*?(\.net)', ' ', doc)
+    doc = re.sub('(Nguồn)\s*?(https:\/\/).*?(\.vgp)', ' ', doc)
+    doc = re.sub('(Nguồn)\s*?(http:\/\/).*?(\.vgp)', ' ', doc)
+
+    doc = re.sub('(http:\/\/).*?(\.htm)', ' ', doc)
+    doc = re.sub('(http:\/\/).*?(\.html)', ' ', doc)
+    doc = re.sub('(https:\/\/).*?(\/\/)', ' ', doc)
+    doc = re.sub('(https:\/\/).*?(\.htm)', ' ', doc)
+    doc = re.sub('(https:\/\/).*?(\.html)', ' ', doc)
+    doc = re.sub('(https:\/\/).*?(\.vn)', ' ', doc)
+    doc = re.sub('(https:\/\/).*?(\.net)', ' ', doc)
+    doc = re.sub('(https:\/\/).*?(\.vgp)', ' ', doc)
+    doc = re.sub('(http:\/\/).*?(\.vgp)', ' ', doc)
+    # escape sequence
+    doc = re.sub('\n', ' ', doc)
+    doc = re.sub('\t', ' ', doc)
+    doc = re.sub('\r', ' ', doc)
+
+    doc = normalize_text(doc)
+    return doc
+
+
+def data_cleaning(docs):
+    res = []
+    for d in docs:
+        if 'message' in d:
+            # css and js
+            doc = re.sub('<.*?>', '', d['message'])
+            doc = re.sub('(function).*}', ' ', doc)
+
+            # link
+            doc = re.sub('(Nguồn)\s*?(http:\/\/).*?(\.htm)', ' ', doc)
+            doc = re.sub('(Nguồn)\s*?(http:\/\/).*?(\.html)', ' ', doc)
+            doc = re.sub('(Nguồn)\s*?(https:\/\/).*?(\/\/)', ' ', doc)
+            doc = re.sub('(Nguồn)\s*?(https:\/\/).*?(\.htm)', ' ', doc)
+            doc = re.sub('(Nguồn)\s*?(https:\/\/).*?(\.html)', ' ', doc)
+            doc = re.sub('(Nguồn)\s*?(https:\/\/).*?(\.vn)', ' ', doc)
+            doc = re.sub('(Nguồn)\s*?(https:\/\/).*?(\.net)', ' ', doc)
+            doc = re.sub('(Nguồn)\s*?(https:\/\/).*?(\.vgp)', ' ', doc)
+            doc = re.sub('(Nguồn)\s*?(http:\/\/).*?(\.vgp)', ' ', doc)
+
+            doc = re.sub('(http:\/\/).*?(\.htm)', ' ', doc)
+            doc = re.sub('(http:\/\/).*?(\.html)', ' ', doc)
+            doc = re.sub('(https:\/\/).*?(\/\/)', ' ', doc)
+            doc = re.sub('(https:\/\/).*?(\.htm)', ' ', doc)
+            doc = re.sub('(https:\/\/).*?(\.html)', ' ', doc)
+            doc = re.sub('(https:\/\/).*?(\.vn)', ' ', doc)
+            doc = re.sub('(https:\/\/).*?(\.net)', ' ', doc)
+            doc = re.sub('(https:\/\/).*?(\.vgp)', ' ', doc)
+            doc = re.sub('(http:\/\/).*?(\.vgp)', ' ', doc)
+            # escape sequence
+            doc = re.sub('\n', ' ', doc)
+            doc = re.sub('\t', ' ', doc)
+            doc = re.sub('\r', ' ', doc)
+
+            d['message'] = doc
+            res.append(d)
+    return res
+
+
+def segment(docs, lang="vi"):
+    segmented_docs = []
+    for d in docs:
+        # if len(d.get('message', "")) > 8000 or len(d.get('message', "")) < 100:
+        if len(d.get('message', "")) > 8000:
+            continue
+        if 'snippet' not in d:
+            continue
+        try:
+            if lang == "vi":
+                snippet = d.get('snippet', "")
+                segmented_snippet = ""
+                segmented_sentences_snippet = annotator.tokenize(snippet)
+                for sentence in segmented_sentences_snippet:
+                    segmented_snippet += ' ' + ' '.join(sentence)
+                segmented_snippet = segmented_snippet.replace('\xa0', '')
+                d['segmented_snippet'] = segmented_snippet
+            segmented_docs.append(d)
+        except Exception:
+            pass
+    return segmented_docs
+
+
+def timestamp_to_date(timestamp):
+    return datetime.datetime.fromtimestamp(timestamp).strftime('%d/%m/%Y')
+
+
+def post_processing(response, top_cluster=5, top_sentence=5, topn_summary=5, sorted_field='', max_doc_per_cluster = 50, delete_message=True):
+    print(f'[INFO] sorted_field: {sorted_field}')
+    MAX_DOC_PER_CLUSTER = max_doc_per_cluster
+
+    lst_ids = []
+    lst_top = []
+    lst_res = []
+    for i in response:
+        lst_ids.append(i)
+
+        if  not sorted_field.strip():
+            lst_top.append(len(response[i]))
+        else:
+            lst_top.append(response[i][0]['max_score'])
+        
+    idx = np.argsort(np.array(lst_top))[::-1] 
+    if top_cluster == -1:
+        top_cluster = len(idx)
+    for i in idx[: top_cluster]:
+        ik = lst_ids[i]
+        if top_sentence == -1:
+            top_sentence = len(response[ik])
+        lst_check_title = []
+        lst_check_not_title = []
+        i_c_t = 0
+        for resss in response[ik]:
+            r_title = resss.get("title", "")
+            if r_title and not r_title.endswith("..."):
+                lst_check_title.append(resss)
+                i_c_t += 1
+            else:
+                lst_check_not_title.append(resss)
+            if i_c_t == top_sentence:
+                break
+        if i_c_t == top_sentence:
+            lst_res.append(lst_check_title)
+        else:
+            lst_check_title.extend(lst_check_not_title)
+            lst_res.append(lst_check_title[:top_sentence])
+        #lst_res.append(response[ik][:top_sentence])
+    dict_res = {}
+    for i in range(len(lst_res)):
+        dict_res[str(i + 1)] = lst_res[i][:MAX_DOC_PER_CLUSTER]
+        for j in range(min(len(dict_res[str(i + 1)]), 3)):
+            dict_res[str(i + 1)][0]["title_summarize"].append(dict_res[str(i + 1)][j].get("snippet", ""))
+        summary_text = get_summary_bert(dict_res[str(i + 1)][0].get("message", ""), lang=get_summary_bert(dict_res[str(i + 1)][0].get("lang", "vi"), topn=topn_summary, title=dict_res[str(i + 1)][0].get("title", ""), snippet=dict_res[str(i + 1)][0].get("snippet", ""))
+        if len(summary_text) < 10:
+            summary_text = dict_res[str(i + 1)][0].get("snippet", "")
+            if len(summary_text) < 10:
+                summary_text = dict_res[str(i + 1)][0].get("title", "")
+        dict_res[str(i + 1)][0]["content_summary"] = utils.remove_image_keyword(summary_text)
+        kew_phares = []
+        dict_res[str(i + 1)][0]["topic_keywords"] = kew_phares
+
+        if delete_message:
+            for j in range(len(dict_res[str(i + 1)])):
+                if "message" in dict_res[str(i + 1)][j]:
+                    del dict_res[str(i + 1)][j]["message"]
+    return dict_res
+
+
+def get_lang(docs):
+    lang_vi = 0
+    lang_en = 0
+    for d in docs:
+        if d.get("lang", "") == "vi":
+            lang_vi += 1
+        else:
+            lang_en += 1
+    if lang_vi >= lang_en:
+        return "vi"
+    return "en"
+
+
+# def topic_clustering(docs, distance_threshold, top_cluster=5, top_sentence=5, topn_summary=5, sorted_field=''):
+#     global model, model_en
+#     docs = docs[:30000]
+#     lang = get_lang(docs)
+#     result = {}
+#     docs = segment(docs, lang=lang)
+#     if len(docs) < 2:
+#         return result
+#     if lang == "vi":
+#         features = [doc.get('title', "") + ". " + doc.get('snippet', "") for doc in docs]
+#         vectors = model.encode(features, show_progress_bar=False)
+#     else:
+#         features = [doc.get('title', "") + ". " + doc.get('snippet', "") for doc in docs]
+#         vectors = model_en.encode(features, show_progress_bar=False)
+#     clusteror = AgglomerativeClustering(n_clusters=None, compute_full_tree=True, affinity='cosine',
+#                                         linkage='single', distance_threshold=distance_threshold)
+#     clusteror.fit(vectors)
+#     print(clusteror.n_clusters_)
+#     for i in range(clusteror.n_clusters_):
+#         result[str(i + 1)] = []
+#     for i in range(len(clusteror.labels_)):
+#         cluster_no = clusteror.labels_[i]   
+#         if docs[i].get('domain','') not in ["cungcau.vn","baomoi.com","news.skydoor.net"]:
+#             response_doc = {}
+#             response_doc = docs[i]
+#             if 'domain' in docs[i]:
+#                 response_doc['domain'] = docs[i]['domain']
+#             if 'url' in docs[i]:
+#                 response_doc['url'] = docs[i]['url']
+#             if 'title' in docs[i]:
+#                 response_doc['title'] = clean_text(docs[i]['title'])
+#             if 'snippet' in docs[i]:
+#                 response_doc['snippet'] = clean_text(docs[i]['snippet'])
+#             if 'created_time' in docs[i]:
+#                 response_doc['created_time'] = docs[i]['created_time']
+#             if 'message' in docs[i]:
+#                 title = docs[i].get('title','')
+#                 snippet = docs[i].get('snippet','')
+#                 message = docs[i].get('message','')
+#                 if title.strip():
+#                     split_mess = message.split(title)
+#                     if len(split_mess) > 1:
+#                         message = title.join(split_mess[1:])
+#                 if snippet.strip():
+#                     split_mess = message.split(snippet)
+#                     if len(split_mess) > 1:
+#                         message = snippet.join(split_mess[1:])
+
+#                 response_doc['message'] = clean_text(message)
+#             if 'id' in docs[i]:
+#                 response_doc['id'] = docs[i]['id']
+#             response_doc['score'] = 0.0
+#             response_doc['title_summarize'] = []
+#             response_doc['content_summary'] = ""
+#             response_doc['total_facebook_viral'] = 0
+#             result[str(cluster_no + 1)].append(response_doc)
+    
+#     empty_clus_ids = []
+#     for x in result:
+#         result[x] = sorted(result[x], key=lambda i: -len(i.get('message','')))
+#         if len( result[x]) > 0:
+#             if len(result[x]) > 1:
+#                 result[x] = check_duplicate_title_domain(result[x])
+#             result[x][0]['num_docs'] = len(result[x])
+#         else:
+#             empty_clus_ids.append(x)
+    
+#     for x in empty_clus_ids:
+#         result.pop(x,None)
+#     # result = dict(sorted(result.items(), key=lambda i: -len(i[1])))[:top_cluster]
+#     return post_processing(result, top_cluster=top_cluster, top_sentence=top_sentence, topn_summary=topn_summary, sorted_field = sorted_field)
+
+def topic_clustering(docs, distance_threshold, top_cluster=5, top_sentence=5, topn_summary=5, sorted_field='', max_doc_per_cluster=50, delete_message=True):
+    global model, model_en
+    docs = docs[:30000]
+    lang = get_lang(docs)
+    result = {}
+    cluster_score = {}
+    docs = segment(docs, lang=lang)
+    if len(docs) < 2:
+        return result
+    if lang == "vi":
+        features = [doc.get('title', "") + ". " + doc.get('snippet', "") for doc in docs]
+        vectors = model.encode(features, show_progress_bar=False)
+    else:
+        features = [doc.get('title', "") + ". " + doc.get('snippet', "") for doc in docs]
+        vectors = model_en.encode(features, show_progress_bar=False)
+    clusteror = AgglomerativeClustering(n_clusters=None, compute_full_tree=True, affinity='cosine',
+                                        linkage='single', distance_threshold=distance_threshold)
+    clusteror.fit(vectors)
+    print(clusteror.n_clusters_)
+    for i in range(clusteror.n_clusters_):
+        result[str(i + 1)] = []
+        cluster_score[str(i + 1)] = 0
+    for i in range(len(clusteror.labels_)):
+        cluster_no = clusteror.labels_[i]   
+        if docs[i].get('domain','') not in ["cungcau.vn","baomoi.com","news.skydoor.net"]:
+            response_doc = {}
+            response_doc = docs[i]
+            score = response_doc.get('score', 0)
+            if not docs[i].get('message','').strip():
+                continue 
+            if score > cluster_score[str(cluster_no + 1)]:
+                cluster_score[str(cluster_no + 1)] = score
+            if 'domain' in docs[i]:
+                response_doc['domain'] = docs[i]['domain']
+            if 'url' in docs[i]:
+                response_doc['url'] = docs[i]['url']
+            if 'title' in docs[i]:
+                response_doc['title'] = clean_text(docs[i]['title'])
+            if 'snippet' in docs[i]:
+                response_doc['snippet'] = clean_text(docs[i]['snippet'])
+            if 'created_time' in docs[i]:
+                response_doc['created_time'] = docs[i]['created_time']
+            if 'message' in docs[i]:
+                title = docs[i].get('title','')
+                snippet = docs[i].get('snippet','')
+                message = docs[i].get('message','')
+                if title.strip():
+                    split_mess = message.split(title)
+                    if len(split_mess) > 1:
+                        message = title.join(split_mess[1:])
+                if snippet.strip():
+                    split_mess = message.split(snippet)
+                    if len(split_mess) > 1:
+                        message = snippet.join(split_mess[1:])
+
+                response_doc['message'] = clean_text(message)
+            if 'id' in docs[i]:
+                response_doc['id'] = docs[i]['id']
+            # response_doc['score'] = 0.0
+            response_doc['title_summarize'] = []
+            response_doc['content_summary'] = ""
+            response_doc['total_facebook_viral'] = 0
+            result[str(cluster_no + 1)].append(response_doc)
+    
+    empty_clus_ids = []
+    for x in result:
+        result[x] = sorted(result[x], key=lambda i: -len(i.get('message','')))
+        if len( result[x]) > 0:
+            if len(result[x]) > 1:
+                result[x] = check_duplicate_title_domain(result[x])
+            result[x][0]['num_docs'] = len(result[x])
+            result[x][0]['max_score'] = cluster_score[x]
+        else:
+            empty_clus_ids.append(x)
+    
+    for x in empty_clus_ids:
+        result.pop(x,None)
+    # result = dict(sorted(result.items(), key=lambda i: -len(i[1])))[:top_cluster]
+    return post_processing(result, top_cluster=top_cluster, top_sentence=top_sentence, topn_summary=topn_summary, sorted_field = sorted_field, max_doc_per_cluster=max_doc_per_cluster, delete_message=delete_message)
+
+def check_duplicate_title_domain(docs):
+    lst_title_domain = [f"{d.get('domain', '')} {d.get('title','')}" for d in docs]
+    for i in range(1,len(lst_title_domain) -1):
+        for j in range(i+1,len(lst_title_domain)):
+            if lst_title_domain[j] == lst_title_domain[i]:
+                lst_title_domain[j] = 'dup'
+    lst_filter_docs = [docs[i] for i,x in enumerate(lst_title_domain) if x != 'dup']
+    return lst_filter_docs
+def convert_date(text):
+    text = text.replace(".", "/")
+    text = text.replace("-", "/")
+    return text
+
+
+def check_keyword(sentence):
+    keyword = ['sáng', 'trưa', 'chiều', 'tối', 'đến', 'hôm', 'ngày', 'tới']
+    for k in keyword:
+        if k in sentence:
+            return True
+    return False
+
+
+def extract_events_and_time(docs, publish_date):
+    def standardize(date_str):
+        return date_str.replace('.', '/').replace('-', '/')
+
+    def add_0(date_str):
+
+        date_str = date_str.split('/')
+        res = []
+        for o in date_str:
+            o = re.sub('\s+', '', o)
+            if len(o) < 2:
+                o = '0' + o
+            res.append(o)
+        date_str = '/'.join(res)
+        return date_str
+
+    def get_date_list(reg, sentence):
+        find_object = re.finditer(reg, sentence)
+        date_list = [x.group() for x in find_object]
+        return date_list
+
+    year = publish_date.split('/')[2]
+
+    # dd/mm/yyyy
+    reg_exp_1 = '(\D|^)(?:0?[1-9]|[12][0-9]|3[01])[- \/.](?:0?[1-9]|1[012])[- \/.]([12]([0-9]){3})(\D|$)'
+    # #mm/yyyy
+    # reg_exp_5 = '(\D|^)(?:0?[1-9]|1[012])[- \/.]([12]([0-9]){3})(\D|$)'
+    # dd/mm
+    reg_exp_2 = '(\D|^)(?:0?[1-9]|[12][0-9]|3[01])[- \/.](?:0?[1-9]|1[012])(\D|$)'
+
+    # ngày  dd tháng mm năm yyyy
+    reg_exp_3 = '(ngày)\s*\d{1,2}\s*(tháng)\s*\d{1,2}\s*(năm)\s*\d{4}'
+    # ngày dd tháng mm
+    reg_exp_4 = '(ngày)\s*\d{1,2}\s*(tháng)\s*\d{1,2}'
+
+    result = []
+    for d in docs:
+        text = d['message']
+        for sentence in sent_tokenize(text):
+            lower_sentence = sentence.lower()
+            c = re.search(reg_exp_3, sentence.lower())
+            d = re.search(reg_exp_4, sentence.lower())
+            # e = re.search(reg_exp_5, sentence.lower())
+            a = re.search(reg_exp_1, sentence)
+            b = re.search(reg_exp_2, sentence)
+            #
+            if (a or b or c or d) and check_keyword(lower_sentence):
+                date_list = get_date_list(reg_exp_1, lower_sentence)
+                date_entity = ''
+                if date_list:
+                    date_entity = add_0(standardize(date_list[0]))
+                elif get_date_list(reg_exp_2, lower_sentence):
+                    date_list = get_date_list(reg_exp_2, lower_sentence)
+                    date_entity = add_0(standardize(date_list[0]) + '/' + year)
+                elif get_date_list(reg_exp_3, lower_sentence):
+                    date_list = get_date_list(reg_exp_3, lower_sentence)
+
+                    date_entity = date_list[0].replace('ngày', '').replace('tháng', '').replace('năm', '').strip()
+                    date_entity = re.sub('\s+', ' ', date_entity)
+                    date_entity = date_entity.replace(' ', '/')
+                    date_entity = add_0(date_entity)
+                else:
+                    date_list = get_date_list(reg_exp_4, lower_sentence)
+                    if date_list != []:
+                        date_entity = date_list[0].replace('ngày', '').replace('tháng', '').replace('năm', '').strip()
+                        date_entity = re.sub('\s+', ' ', date_entity)
+                        date_entity = date_entity.replace(' ', '/')
+                        date_entity = date_entity + '/' + year
+                        date_entity = add_0(date_entity)
+                result.append((sentence, date_entity))
+    return result

function/topic_clustering_social.py

@@ -0,0 +1,156 @@
+import json
+import time
+from .utils import get_sbert_embedding, clean_text
+from sklearn.cluster import AgglomerativeClustering
+from sklearn.metrics.pairwise import cosine_similarity
+from nltk import sent_tokenize
+import requests
+# from clean_text import normalize_text
+
+MAX_LENGTH_FEATURE = 250
+MIN_LENGTH_FEATURE = 100
+URL_CHECK_SPAM = "http://10.9.3.70:30036/predict"
+
+def check_spam(docs):
+    json_body = {
+        "domain_id": "",
+        "records": [
+            {
+            "text": doc.get("message",""),
+            "idxcol": 1
+            } for doc in docs
+        ] 
+    }
+
+    result = requests.post(URL_CHECK_SPAM, json = json_body).json()
+    docs = [x for i,x in enumerate(docs) if result[i]["label"] == 0]
+    return docs
+
+def preocess_feature(doc):
+    message = doc.get("message","")
+    paras = message.split("\n") 
+    feature = ""
+    paras = [clean_text(x.strip(), normalize=False) for x in paras if x.strip() and len(x.strip()) > 10]
+    for para in paras:
+        if len(feature) + len(para) < MAX_LENGTH_FEATURE:
+            feature += " " +para
+        elif len(feature) < MIN_LENGTH_FEATURE:
+            sens = sent_tokenize(para)
+            for sen in sens:
+                if len(feature) + len(sen) < MAX_LENGTH_FEATURE or len(feature.strip()) < MIN_LENGTH_FEATURE:
+                    feature += " " +sen
+    return feature
+
+def topic_clustering(docs, distance_threshold, top_cluster=5, top_sentence=5, topn_summary=5, sorted_field='', max_doc_per_cluster=50, delete_message=True, is_check_spam = True):
+    # global model, model_en
+    
+    docs = [x for x in docs if len(x.get("message","")) > 100]
+    docs = docs[:30000]
+    if is_check_spam:
+        docs = check_spam(docs)
+    result = {}
+    cluster_score = {}
+
+    t1 = time.time()
+    if len(docs) < 1:
+        return result
+    elif len(docs) == 1:
+        return {
+            "0": docs
+        }
+
+    # features = [doc.get('title', "") + ". " + doc.get('snippet', "") for doc in docs]
+    
+    f_docs = []
+    for x in docs:
+        ft = preocess_feature(x)
+        if len(ft) > MIN_LENGTH_FEATURE:
+            x["title"] = ft
+            f_docs.append(x)
+    docs = f_docs
+
+    features = [x["title"] for x in docs ]
+    # with open("feature", 'w') as f:
+    #     json.dump(features, f, ensure_ascii = False)
+    # print(features)
+    vectors = get_sbert_embedding(features)
+
+    clusteror = AgglomerativeClustering(n_clusters=None, compute_full_tree=True, affinity='cosine',
+                                        linkage='complete', distance_threshold=distance_threshold)
+    clusteror.fit(vectors)
+    print(f"Time encode + clustering: {time.time() - t1} {clusteror.n_clusters_}")
+    for i in range(clusteror.n_clusters_):
+        result[str(i + 1)] = []
+        cluster_score[str(i + 1)] = 0
+    for i in range(len(clusteror.labels_)):
+        cluster_no = clusteror.labels_[i]   
+        if docs[i].get('domain','') not in ["cungcau.vn","baomoi.com","news.skydoor.net"]:
+            response_doc = {}
+            response_doc = docs[i]
+            score = response_doc.get('score', 0)
+            if not docs[i].get('message','').strip():
+                continue 
+            if score > cluster_score[str(cluster_no + 1)]:
+                cluster_score[str(cluster_no + 1)] = score
+            if 'domain' in docs[i]:
+                response_doc['domain'] = docs[i]['domain']
+            if 'url' in docs[i]:
+                response_doc['url'] = docs[i]['url']
+            if 'title' in docs[i]:
+                response_doc['title'] = clean_text(docs[i]['title'])
+            if 'snippet' in docs[i]:
+                response_doc['snippet'] = clean_text(docs[i]['snippet'])
+            if 'created_time' in docs[i]:
+                response_doc['created_time'] = docs[i]['created_time']
+            if "sentiment" in docs[i]:
+                response_doc['sentiment'] = docs[i]['sentiment']
+            if 'message' in docs[i]:
+                title = docs[i].get('title','')
+                snippet = docs[i].get('snippet','')
+                message = docs[i].get('message','')
+                # if title.strip():
+                #     split_mess = message.split(title)
+                #     if len(split_mess) > 1:
+                #         message = title.join(split_mess[1:])
+                # if snippet.strip():
+                #     split_mess = message.split(snippet)
+                #     if len(split_mess) > 1:
+                #         message = snippet.join(split_mess[1:])
+
+                response_doc['message'] = clean_text(message)
+            if 'id' in docs[i]:
+                response_doc['id'] = docs[i]['id']
+            # response_doc['score'] = 0.0
+
+            # response_doc['title_summarize'] = []
+            # response_doc['content_summary'] = ""
+            # response_doc['total_facebook_viral'] = 0
+            result[str(cluster_no + 1)].append(response_doc)
+    
+    empty_clus_ids = []
+    for x in result:
+        result[x] = sorted(result[x], key=lambda i: -len(i.get('message','')))
+        if len( result[x]) > 0:
+            # if len(result[x]) > 1:
+            #     result[x] = check_duplicate_title_domain(result[x])
+            result[x][0]['num_docs'] = len(result[x])
+            result[x][0]['max_score'] = cluster_score[x]
+        else:
+            empty_clus_ids.append(x)
+    
+    for x in empty_clus_ids:
+        result.pop(x,None)
+    
+    result = dict( sorted(result.items(), key=lambda i: -len(i[1]))[:top_cluster])
+    return result
+    # return post_processing(result, top_cluster=top_cluster, top_sentence=top_sentence, topn_summary=topn_summary, sorted_field = sorted_field, max_doc_per_cluster=max_doc_per_cluster, delete_message=delete_message)
+
+if __name__ == '__main__':
+    # with open("/home2/vietle/DA-Report/social.json", 'r') as f:
+    #     docs = json.load(f)[:2000]
+    with open("/home2/vietle/news-cms/topic_summarization/data/news_cms.social.json", 'r') as f:
+        docs = json.load(f)[:10000]
+    clusters = topic_clustering(docs, distance_threshold=0.2, top_cluster=5000, top_sentence=5, topn_summary=5, sorted_field='', max_doc_per_cluster=50, delete_message=False)
+    with open("/home2/vietle/news-cms/topic_summarization/cluster/news_cms.social.json", 'w') as f:
+
+        json.dump(clusters,f, ensure_ascii =False)

function/topic_clustering_v2.py

@@ -0,0 +1,390 @@
+from tensorRT import inference
+import re
+from vncorenlp import VnCoreNLP
+from nltk.tokenize import sent_tokenize
+import torch
+import datetime
+from sklearn.cluster import AgglomerativeClustering
+from sklearn.metrics.pairwise import cosine_similarity
+import numpy as np
+import json
+from . import utils
+import time
+from summary import text_summary, get_summary_bert
+from function.clean_text import normalize_text
+
+# from . import detect_time as dt
+
+device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+use_cuda = torch.cuda.is_available()
+print(torch.cuda.is_available())
+
+# annotator = VnCoreNLP('vncorenlp/VnCoreNLP-1.1.1.jar', port=9191, annotators="wseg,pos", max_heap_size='-Xmx2g')
+
+
+def detect_postaging(text_in):
+    word_segmented_text = annotator.annotate(text_in)
+    lst_k = []
+    for se in word_segmented_text["sentences"]:
+        for kw in se:
+            if kw["posTag"] in ("Np", "Ny", "N"):
+                if kw["posTag"] == "N" and "_" not in kw["form"]:
+                    continue
+                lst_k.append(kw["form"].replace("_", " "))
+    return list(set(lst_k))
+
+def clean_text(text_in):
+    doc = re.sub('<.*?>', '', text_in)
+    doc = re.sub('(function).*}', ' ', doc)
+    # link
+    doc = re.sub('(Nguồn)\s*?(http:\/\/).*?(\.htm)', ' ', doc)
+    doc = re.sub('(Nguồn)\s*?(http:\/\/).*?(\.html)', ' ', doc)
+    doc = re.sub('(Nguồn)\s*?(https:\/\/).*?(\/\/)', ' ', doc)
+    doc = re.sub('(Nguồn)\s*?(https:\/\/).*?(\.htm)', ' ', doc)
+    doc = re.sub('(Nguồn)\s*?(https:\/\/).*?(\.html)', ' ', doc)
+    doc = re.sub('(Nguồn)\s*?(https:\/\/).*?(\.vn)', ' ', doc)
+    doc = re.sub('(Nguồn)\s*?(https:\/\/).*?(\.net)', ' ', doc)
+    doc = re.sub('(Nguồn)\s*?(https:\/\/).*?(\.vgp)', ' ', doc)
+    doc = re.sub('(Nguồn)\s*?(http:\/\/).*?(\.vgp)', ' ', doc)
+
+    doc = re.sub('(http:\/\/).*?(\.htm)', ' ', doc)
+    doc = re.sub('(http:\/\/).*?(\.html)', ' ', doc)
+    doc = re.sub('(https:\/\/).*?(\/\/)', ' ', doc)
+    doc = re.sub('(https:\/\/).*?(\.htm)', ' ', doc)
+    doc = re.sub('(https:\/\/).*?(\.html)', ' ', doc)
+    doc = re.sub('(https:\/\/).*?(\.vn)', ' ', doc)
+    doc = re.sub('(https:\/\/).*?(\.net)', ' ', doc)
+    doc = re.sub('(https:\/\/).*?(\.vgp)', ' ', doc)
+    doc = re.sub('(http:\/\/).*?(\.vgp)', ' ', doc)
+    # escape sequence
+    doc = re.sub('\n', ' ', doc)
+    doc = re.sub('\t', ' ', doc)
+    doc = re.sub('\r', ' ', doc)
+
+    doc = normalize_text(doc)
+    return doc
+
+
+def data_cleaning(docs):
+    res = []
+    for d in docs:
+        if 'message' in d:
+            # css and js
+            doc = re.sub('<.*?>', '', d['message'])
+            doc = re.sub('(function).*}', ' ', doc)
+
+            # link
+            doc = re.sub('(Nguồn)\s*?(http:\/\/).*?(\.htm)', ' ', doc)
+            doc = re.sub('(Nguồn)\s*?(http:\/\/).*?(\.html)', ' ', doc)
+            doc = re.sub('(Nguồn)\s*?(https:\/\/).*?(\/\/)', ' ', doc)
+            doc = re.sub('(Nguồn)\s*?(https:\/\/).*?(\.htm)', ' ', doc)
+            doc = re.sub('(Nguồn)\s*?(https:\/\/).*?(\.html)', ' ', doc)
+            doc = re.sub('(Nguồn)\s*?(https:\/\/).*?(\.vn)', ' ', doc)
+            doc = re.sub('(Nguồn)\s*?(https:\/\/).*?(\.net)', ' ', doc)
+            doc = re.sub('(Nguồn)\s*?(https:\/\/).*?(\.vgp)', ' ', doc)
+            doc = re.sub('(Nguồn)\s*?(http:\/\/).*?(\.vgp)', ' ', doc)
+
+            doc = re.sub('(http:\/\/).*?(\.htm)', ' ', doc)
+            doc = re.sub('(http:\/\/).*?(\.html)', ' ', doc)
+            doc = re.sub('(https:\/\/).*?(\/\/)', ' ', doc)
+            doc = re.sub('(https:\/\/).*?(\.htm)', ' ', doc)
+            doc = re.sub('(https:\/\/).*?(\.html)', ' ', doc)
+            doc = re.sub('(https:\/\/).*?(\.vn)', ' ', doc)
+            doc = re.sub('(https:\/\/).*?(\.net)', ' ', doc)
+            doc = re.sub('(https:\/\/).*?(\.vgp)', ' ', doc)
+            doc = re.sub('(http:\/\/).*?(\.vgp)', ' ', doc)
+            # escape sequence
+            doc = re.sub('\n', ' ', doc)
+            doc = re.sub('\t', ' ', doc)
+            doc = re.sub('\r', ' ', doc)
+
+            d['message'] = doc
+            res.append(d)
+    return res
+
+
+def segment(docs, lang="vi"):
+    segmented_docs = []
+    for d in docs:
+        print(d)
+        # if len(d.get('message', "")) > 8000 or len(d.get('message', "")) < 100:
+        if len(d.get('message', "")) > 8000:
+            continue
+        if 'snippet' not in d:
+            continue
+        try:
+            if lang == "vi":
+                snippet = d.get('snippet', "")
+                segmented_snippet = ""
+                segmented_sentences_snippet = annotator.tokenize(snippet)
+                for sentence in segmented_sentences_snippet:
+                    segmented_snippet += ' ' + ' '.join(sentence)
+                segmented_snippet = segmented_snippet.replace('\xa0', '')
+                d['segmented_snippet'] = segmented_snippet
+            segmented_docs.append(d)
+        except Exception:
+            pass
+    return segmented_docs
+
+
+def timestamp_to_date(timestamp):
+    return datetime.datetime.fromtimestamp(timestamp).strftime('%d/%m/%Y')
+
+
+def post_processing(response, top_cluster=5, top_sentence=5, topn_summary=5, sorted_field='', max_doc_per_cluster = 50, delete_message=True):
+    print(f'[INFO] sorted_field: {sorted_field}')
+    MAX_DOC_PER_CLUSTER = max_doc_per_cluster
+
+    lst_ids = []
+    lst_top = []
+    lst_res = []
+    for i in response:
+        lst_ids.append(i)
+
+        if  not sorted_field.strip():
+            lst_top.append(len(response[i]))
+        else:
+            lst_top.append(response[i][0]['max_score'])
+        
+    idx = np.argsort(np.array(lst_top))[::-1] 
+    if top_cluster == -1:
+        top_cluster = len(idx)
+    for i in idx[: top_cluster]:
+        ik = lst_ids[i]
+        if top_sentence == -1:
+            top_sentence = len(response[ik])
+        lst_check_title = []
+        lst_check_not_title = []
+        i_c_t = 0
+        for resss in response[ik]:
+            r_title = resss.get("title", "")
+            if r_title and not r_title.endswith("..."):
+                lst_check_title.append(resss)
+                i_c_t += 1
+            else:
+                lst_check_not_title.append(resss)
+            if i_c_t == top_sentence:
+                break
+        if i_c_t == top_sentence:
+            lst_res.append(lst_check_title)
+        else:
+            lst_check_title.extend(lst_check_not_title)
+            lst_res.append(lst_check_title[:top_sentence])
+        #lst_res.append(response[ik][:top_sentence])
+    dict_res = {}
+    for i in range(len(lst_res)):
+        dict_res[str(i + 1)] = lst_res[i][:MAX_DOC_PER_CLUSTER]
+        for j in range(min(len(dict_res[str(i + 1)]), 3)):
+            dict_res[str(i + 1)][0]["title_summarize"].append(dict_res[str(i + 1)][j].get("snippet", ""))
+        summary_text = get_summary_bert(dict_res[str(i + 1)][0].get("message", ""), dict_res[str(i + 1)][0].get("lang", "vi"), topn=topn_summary, title=dict_res[str(i + 1)][0].get("title", ""), snippet=dict_res[str(i + 1)][0].get("snippet", ""))
+        if len(summary_text) < 10:
+            summary_text = dict_res[str(i + 1)][0].get("snippet", "")
+            if len(summary_text) < 10:
+                summary_text = dict_res[str(i + 1)][0].get("title", "")
+        dict_res[str(i + 1)][0]["content_summary"] = utils.remove_image_keyword(summary_text)
+        kew_phares = []
+        dict_res[str(i + 1)][0]["topic_keywords"] = kew_phares
+        
+        print("delete_message: ", delete_message)
+        if delete_message:
+            for j in range(len(dict_res[str(i + 1)])):
+                if "message" in dict_res[str(i + 1)][j]:
+                    del dict_res[str(i + 1)][j]["message"]
+    return dict_res
+
+
+def get_lang(docs):
+    lang_vi = 0
+    lang_en = 0
+    for d in docs:
+        if d.get("lang", "") == "vi":
+            lang_vi += 1
+        else:
+            lang_en += 1
+    if lang_en >= lang_vi:
+        return "en"
+    return "vi"
+
+def topic_clustering(docs, distance_threshold, top_cluster=5, top_sentence=5, topn_summary=5, sorted_field='', max_doc_per_cluster=50, delete_message=True):
+    # global model, model_en
+    docs = docs[:30000]
+    lang = get_lang(docs)
+    result = {}
+    cluster_score = {}
+    # docs = segment(docs, lang=lang)
+
+    t1 = time.time()
+    if len(docs) < 1:
+        return result
+    elif len(docs) == 1:
+        return {
+            "0": docs
+        }
+    if lang == "vi":
+        features = [doc.get('title', "") + ". " + doc.get('snippet', "") for doc in docs]
+        vectors = inference.encode(features, lang=lang)
+        # vectors = model.encode(features, show_progress_bar=False)
+    else:
+        features = [doc.get('title', "") + ". " + doc.get('snippet', "") for doc in docs]
+        vectors = inference.encode(features, lang=lang)
+        # vectors = model_en.encode(features, show_progress_bar=False)
+    clusteror = AgglomerativeClustering(n_clusters=None, compute_full_tree=True, affinity='cosine',
+                                        linkage='single', distance_threshold=distance_threshold)
+    clusteror.fit(vectors)
+    print(f"Time encode + clustering: {time.time() - t1} {clusteror.n_clusters_}")
+    for i in range(clusteror.n_clusters_):
+        result[str(i + 1)] = []
+        cluster_score[str(i + 1)] = 0
+    for i in range(len(clusteror.labels_)):
+        cluster_no = clusteror.labels_[i]   
+        if docs[i].get('domain','') not in ["cungcau.vn","baomoi.com","news.skydoor.net"]:
+            response_doc = {}
+            response_doc = docs[i]
+            score = response_doc.get('score', 0)
+            if not docs[i].get('message','').strip():
+                continue 
+            if score > cluster_score[str(cluster_no + 1)]:
+                cluster_score[str(cluster_no + 1)] = score
+            if 'domain' in docs[i]:
+                response_doc['domain'] = docs[i]['domain']
+            if 'url' in docs[i]:
+                response_doc['url'] = docs[i]['url']
+            if 'title' in docs[i]:
+                response_doc['title'] = clean_text(docs[i]['title'])
+            if 'snippet' in docs[i]:
+                response_doc['snippet'] = clean_text(docs[i]['snippet'])
+            if 'created_time' in docs[i]:
+                response_doc['created_time'] = docs[i]['created_time']
+            if "sentiment" in docs[i]:
+                response_doc['sentiment'] = docs[i]['sentiment']
+            if 'message' in docs[i]:
+                title = docs[i].get('title','')
+                snippet = docs[i].get('snippet','')
+                message = docs[i].get('message','')
+                if title.strip():
+                    split_mess = message.split(title)
+                    if len(split_mess) > 1:
+                        message = title.join(split_mess[1:])
+                if snippet.strip():
+                    split_mess = message.split(snippet)
+                    if len(split_mess) > 1:
+                        message = snippet.join(split_mess[1:])
+
+                response_doc['message'] = clean_text(message)
+            if 'id' in docs[i]:
+                response_doc['id'] = docs[i]['id']
+            # response_doc['score'] = 0.0
+            response_doc['title_summarize'] = []
+            response_doc['content_summary'] = ""
+            response_doc['total_facebook_viral'] = 0
+            result[str(cluster_no + 1)].append(response_doc)
+    
+    empty_clus_ids = []
+    for x in result:
+        result[x] = sorted(result[x], key=lambda i: -len(i.get('message','')))
+        if len( result[x]) > 0:
+            if len(result[x]) > 1:
+                result[x] = check_duplicate_title_domain(result[x])
+            result[x][0]['num_docs'] = len(result[x])
+            result[x][0]['max_score'] = cluster_score[x]
+        else:
+            empty_clus_ids.append(x)
+    
+    for x in empty_clus_ids:
+        result.pop(x,None)
+    # result = dict(sorted(result.items(), key=lambda i: -len(i[1])))[:top_cluster]
+    return post_processing(result, top_cluster=top_cluster, top_sentence=top_sentence, topn_summary=topn_summary, sorted_field = sorted_field, max_doc_per_cluster=max_doc_per_cluster, delete_message=delete_message)
+
+def check_duplicate_title_domain(docs):
+    lst_title_domain = [f"{d.get('domain', '')} {d.get('title','')}" for d in docs]
+    for i in range(1,len(lst_title_domain) -1):
+        for j in range(i+1,len(lst_title_domain)):
+            if lst_title_domain[j] == lst_title_domain[i]:
+                lst_title_domain[j] = 'dup'
+    lst_filter_docs = [docs[i] for i,x in enumerate(lst_title_domain) if x != 'dup']
+    return lst_filter_docs
+def convert_date(text):
+    text = text.replace(".", "/")
+    text = text.replace("-", "/")
+    return text
+
+
+def check_keyword(sentence):
+    keyword = ['sáng', 'trưa', 'chiều', 'tối', 'đến', 'hôm', 'ngày', 'tới']
+    for k in keyword:
+        if k in sentence:
+            return True
+    return False
+
+
+def extract_events_and_time(docs, publish_date):
+    def standardize(date_str):
+        return date_str.replace('.', '/').replace('-', '/')
+
+    def add_0(date_str):
+
+        date_str = date_str.split('/')
+        res = []
+        for o in date_str:
+            o = re.sub('\s+', '', o)
+            if len(o) < 2:
+                o = '0' + o
+            res.append(o)
+        date_str = '/'.join(res)
+        return date_str
+
+    def get_date_list(reg, sentence):
+        find_object = re.finditer(reg, sentence)
+        date_list = [x.group() for x in find_object]
+        return date_list
+
+    year = publish_date.split('/')[2]
+
+    # dd/mm/yyyy
+    reg_exp_1 = '(\D|^)(?:0?[1-9]|[12][0-9]|3[01])[- \/.](?:0?[1-9]|1[012])[- \/.]([12]([0-9]){3})(\D|$)'
+    # #mm/yyyy
+    # reg_exp_5 = '(\D|^)(?:0?[1-9]|1[012])[- \/.]([12]([0-9]){3})(\D|$)'
+    # dd/mm
+    reg_exp_2 = '(\D|^)(?:0?[1-9]|[12][0-9]|3[01])[- \/.](?:0?[1-9]|1[012])(\D|$)'
+
+    # ngày  dd tháng mm năm yyyy
+    reg_exp_3 = '(ngày)\s*\d{1,2}\s*(tháng)\s*\d{1,2}\s*(năm)\s*\d{4}'
+    # ngày dd tháng mm
+    reg_exp_4 = '(ngày)\s*\d{1,2}\s*(tháng)\s*\d{1,2}'
+
+    result = []
+    for d in docs:
+        text = d['message']
+        for sentence in sent_tokenize(text):
+            lower_sentence = sentence.lower()
+            c = re.search(reg_exp_3, sentence.lower())
+            d = re.search(reg_exp_4, sentence.lower())
+            # e = re.search(reg_exp_5, sentence.lower())
+            a = re.search(reg_exp_1, sentence)
+            b = re.search(reg_exp_2, sentence)
+            #
+            if (a or b or c or d) and check_keyword(lower_sentence):
+                date_list = get_date_list(reg_exp_1, lower_sentence)
+                date_entity = ''
+                if date_list:
+                    date_entity = add_0(standardize(date_list[0]))
+                elif get_date_list(reg_exp_2, lower_sentence):
+                    date_list = get_date_list(reg_exp_2, lower_sentence)
+                    date_entity = add_0(standardize(date_list[0]) + '/' + year)
+                elif get_date_list(reg_exp_3, lower_sentence):
+                    date_list = get_date_list(reg_exp_3, lower_sentence)
+
+                    date_entity = date_list[0].replace('ngày', '').replace('tháng', '').replace('năm', '').strip()
+                    date_entity = re.sub('\s+', ' ', date_entity)
+                    date_entity = date_entity.replace(' ', '/')
+                    date_entity = add_0(date_entity)
+                else:
+                    date_list = get_date_list(reg_exp_4, lower_sentence)
+                    if date_list != []:
+                        date_entity = date_list[0].replace('ngày', '').replace('tháng', '').replace('năm', '').strip()
+                        date_entity = re.sub('\s+', ' ', date_entity)
+                        date_entity = date_entity.replace(' ', '/')
+                        date_entity = date_entity + '/' + year
+                        date_entity = add_0(date_entity)
+                result.append((sentence, date_entity))
+    return result

function/translate.py

@@ -0,0 +1,37 @@
+from langdetect import detect
+import requests
+import json
+import time
+
+URL_TRANSLATOR = "http://10.9.3.241:8093/translator"
+def detect_lang(text):
+    try:
+        lang = detect(text)
+    except:
+        lang = 'en'
+    return lang
+
+def translate_text_multi_layer(source, target, text, url = URL_TRANSLATOR):
+    if source == "":
+        source = detect_lang(text)
+    print("PPPPPPPPPPPPP")
+    if not text.strip() or source == target:
+        return text
+    
+    json_body = {
+        "doc": text,
+        "src_lang": source,
+        "tgt_lang": target
+    }
+    print("CCCCCCCCCCCC")
+    res= requests.post(url, json=json_body)
+    print("translate: ", res.status_code)
+    path_log = f"log_tran/requests_tran_{time.time()}.txt"
+    with open(path_log, "w", encoding="utf-8") as f:
+        f.write(json.dumps(json_body) + "\n")
+    if res.status_code == 200:
+        res = res.json()
+        with open(path_log, "a", encoding="utf-8") as f:
+            f.write(json.dumps(res) + "\n")
+        return res
+    return ''

function/utils.py

@@ -0,0 +1,94 @@
+import editdistance
+import requests
+import numpy as np
+import re
+from .clean_text import normalize_text
+URL_SBERT = "http://10.9.3.240:6789/sbert/encode_list"
+# app_config.parse_url_api('api_sbert')
+
+def get_sbert_embedding(lst_sentence, url = URL_SBERT):
+    input_data = {
+        "sentences": lst_sentence
+    }
+    embs = requests.post(url, json=input_data).json()
+    embs = np.array(embs)
+
+    return embs
+
+def is_number(word):
+    lst_end = ['$', '%', 'vnđ', '.', ',']
+    word_lo = word.lower()
+    for k in lst_end:
+        word_lo = word_lo.replace(k, '')
+
+    if word_lo.isdigit():
+        return True
+    else:
+        return False
+
+
+def get_number(text):
+    dt = text.split(' ')
+    for w in dt:
+        if is_number(w):
+            return w
+
+    return ''
+
+
+def check_editdistance(ww1, ww2):
+    if len(ww1) == 0 or len(ww1) == 0:
+        return 0
+    else:
+        n_c = editdistance.eval(ww1.lower(), ww2.lower())
+        score = n_c / max(len(ww1), len(ww2))
+        return 1 - score
+
+
+def remove_image_keyword(text_input):
+    lst_key = ["ảnh:", "ảnh :", "Ảnh:", "Ảnh :",
+               "Ảnh minh họa:", "Ảnh minh họa :", "ảnh minh họa:", "ảnh minh họa :",
+               "Nguồn:", "nguồn:", "Nguồn :", "nguồn :",
+               "Source:", "Source :", "source:", "source :",
+               "Src:", "Src :", "src:", "src :",
+               "Image:", "Image :", "img:", "img :",
+               "image:", "image :", "Img:", "Img :",
+               "xem tiếp", "xem thêm", "Xem tiếp", "Xem thêm"]
+    for k in lst_key:
+        text_input = text_input.replace(k, " ")
+    return text_input.strip()
+
+def clean_text(text_in, normalize=True):
+    doc = re.sub('<.*?>', '', text_in)
+    doc = re.sub('(function).*}', ' ', doc)
+    # link
+    doc = re.sub('(Nguồn)\s*?(http:\/\/).*?(\.htm)', ' ', doc)
+    doc = re.sub('(Nguồn)\s*?(http:\/\/).*?(\.html)', ' ', doc)
+    doc = re.sub('(Nguồn)\s*?(https:\/\/).*?(\/\/)', ' ', doc)
+    doc = re.sub('(Nguồn)\s*?(https:\/\/).*?(\.htm)', ' ', doc)
+    doc = re.sub('(Nguồn)\s*?(https:\/\/).*?(\.html)', ' ', doc)
+    doc = re.sub('(Nguồn)\s*?(https:\/\/).*?(\.vn)', ' ', doc)
+    doc = re.sub('(Nguồn)\s*?(https:\/\/).*?(\.net)', ' ', doc)
+    doc = re.sub('(Nguồn)\s*?(https:\/\/).*?(\.vgp)', ' ', doc)
+    doc = re.sub('(Nguồn)\s*?(http:\/\/).*?(\.vgp)', ' ', doc)
+
+    doc = re.sub('(http:\/\/).*?(\.htm)', ' ', doc)
+    doc = re.sub('(http:\/\/).*?(\.html)', ' ', doc)
+    doc = re.sub('(https:\/\/).*?(\/\/)', ' ', doc)
+    doc = re.sub('(https:\/\/).*?(\.htm)', ' ', doc)
+    doc = re.sub('(https:\/\/).*?(\.html)', ' ', doc)
+    doc = re.sub('(https:\/\/).*?(\.vn)', ' ', doc)
+    doc = re.sub('(https:\/\/).*?(\.net)', ' ', doc)
+    doc = re.sub('(https:\/\/).*?(\.vgp)', ' ', doc)
+    doc = re.sub('(http:\/\/).*?(\.vgp)', ' ', doc)
+    # escape sequence
+    doc = re.sub('\n', ' ', doc)
+    doc = re.sub('\t', ' ', doc)
+    doc = re.sub('\r', ' ', doc)
+
+    if normalize:
+        doc = normalize_text(doc)
+    return doc
+
+if __name__ == '__main__':
+    print(check_editdistance('tttt', 'tt'))

get_config.py

@@ -0,0 +1,5 @@
+from yaml import load, Loader
+
+config_params = {}
+with open('config/config.yml', encoding='utf-8') as f:
+    config_params.update(load(f, Loader=Loader))

iclibs/ic_rabbit.py

@@ -0,0 +1,126 @@
+# -*- coding: utf-8 -*-
+import pika
+import json
+
+
+class ICRabbitMQ(object):
+    def __init__(self, host, virtual_host, usr, passw, **kwargs):
+        """
+        Khởi tạo
+        :param host: địa chỉ rabbitmq server
+        :param virtual_host: virtual_host
+        :param queue_name: tên queue
+        :param usr: user rabbitmq server
+        :param passw: password
+        """
+        self.host = host
+        self.virtual_host = virtual_host
+        self.user = usr
+        self.passw = passw
+        self.credentials = pika.PlainCredentials(usr, passw)
+        self.connection = None
+        self.kwargs = kwargs
+
+    def init_connection(self):
+        self.connection = \
+            pika.BlockingConnection(
+                pika.ConnectionParameters(host=self.host, virtual_host=self.virtual_host, credentials=self.credentials))
+
+    def connection_close(self):
+        self.connection.close()
+
+    def connection_status(self):
+        return self.connection.is_open
+
+    def init_queue(self, queue_name, exchange="", exchange_type='fanout', durable=True, max_priority=-1):
+        """
+        khởi tạo queue
+        :param exchange:
+        :param queue_name: tên queue
+        :param durable: true (Queue vẫn tồn tại nếu nhưng RabitMQ khởi động lại)
+        :param max_priority: Mức độ priority tối đa; None thì không xét priority;
+                            khác None thì xét priority (tối đa priority = 10)
+        :return: channel
+        """
+        if self.connection is None:
+            self.init_connection()
+        channel = self.connection.channel()
+        if exchange == "" and queue_name != "":
+            if max_priority == -1:
+                channel.queue_declare(queue=queue_name, durable=durable)
+            else:
+                channel.queue_declare(queue=queue_name, durable=durable, arguments={'x-max-priority': max_priority})
+        else:
+            channel.exchange_declare(exchange=exchange, exchange_type='fanout', durable=durable)
+        return channel
+
+    @staticmethod
+    def publish_message(channel, routing_key, body, priority=-1, delivery_mode=2, exchange=''):
+        """
+        run pushlish message
+        :param channel: channel đã được tạo
+        :param routing_key: key hoặc tên queue (nếu exchange = '')
+        :param body: data push
+        :param priority: mức ưu tiên
+        :param delivery_mode: ??
+        :param exchange: routing
+        """
+        if priority == -1:
+            channel.basic_publish(exchange=exchange, routing_key=routing_key, body=json.dumps(body),
+                                  properties=pika.BasicProperties(delivery_mode=delivery_mode))
+        else:
+            channel.basic_publish(exchange=exchange, routing_key=routing_key, body=json.dumps(body),
+                                  properties=pika.BasicProperties(delivery_mode=delivery_mode, priority=priority))
+        print("push done: ")
+
+    @staticmethod
+    def run_consummer(channel, queue_name, callback_func, is_ack=False):
+        """
+        run consumer
+        :param channel: channel đã được tạo
+        :param queue_name: tên queue
+        :param callback_func: hàm callback được định nghĩa bởi người dùng
+        :return:
+        """
+        print(" *wait message")
+
+        def callback(ch, method, properties, body):
+            body = json.loads(body.decode("utf-8"))
+            if is_ack:
+                ch.basic_ack(delivery_tag=method.delivery_tag)
+                callback_func(body, properties)
+            else:
+                callback_func(body, properties)
+                ch.basic_ack(delivery_tag=method.delivery_tag)
+
+            
+            print("receive done: ")
+
+        channel.basic_qos(prefetch_count=10)
+        channel.basic_consume(queue=queue_name, on_message_callback=callback)
+        channel.start_consuming()
+
+
+if __name__ == '__main__':
+    pass
+    # host, virtual_host, usr, passw = '10.9.2.220', 'camera-detect', 'tuan.dao', 'lEKUWKXwFt'
+    # rab = ICRabbitMQ(host, virtual_host, usr, passw)
+    # queue_name = 'test_rb'
+    #
+    # ## test run producer
+    # channel = rab.init_queue(queue_name)
+    # body_data = {"2": "3423432423"}
+    # ICRabbitMQ.publish_message(channel, queue_name, body_data)
+    #
+    #
+    # ## test run consumer
+    # def callback_func(body):
+    #     """
+    #     function callback người dùng định nghĩa
+    #     :param body: message từ queue
+    #     :return:
+    #     """
+    #     print(body)
+    #
+    #
+    # ICRabbitMQ.run_consummer(channel, queue_name, callback_func)