qingyang's new cas12

.gitattributes

@@ -36,3 +36,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 Human_genes_HUGO_02242024_annotation.txt filter=lfs diff=lfs merge=lfs -text
 SRR25934512.filter.snps.indels.vcf filter=lfs diff=lfs merge=lfs -text
 cas9_model/Cas9_MultiHeadAttention_weights.keras filter=lfs diff=lfs merge=lfs -text
+cas12_model/BiLSTM_Cpf1_weights.keras filter=lfs diff=lfs merge=lfs -text

cas12_model/BiLSTM_Cpf1_weights.keras

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:67a66b6753c9d020d65c018102c4eb5f61a29bdcfb3487ccfd238fcd7bf11d29
+size 4713670

cas12_model/cas12_on_target.ipynb

@@ -0,0 +1 @@
+{"cells":[{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":17019,"status":"ok","timestamp":1714556250362,"user":{"displayName":"qingyang liu","userId":"04216395896980867619"},"user_tz":-120},"id":"6kWEP_7qS7Ct","outputId":"c3e942c3-bc5d-4977-c740-d342fd57a107"},"outputs":[{"output_type":"stream","name":"stdout","text":["Mounted at /content/drive\n"]}],"source":["from google.colab import drive\n","drive.mount('/content/drive')\n","import sys\n","sys.path.append('/content/drive/MyDrive/Colab Notebooks/Cas12')"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"YGBbL_RlTHF_","executionInfo":{"status":"ok","timestamp":1714556268469,"user_tz":-120,"elapsed":15970,"user":{"displayName":"qingyang liu","userId":"04216395896980867619"}},"colab":{"base_uri":"https://localhost:8080/"},"outputId":"abacda11-5b55-4253-a12a-ef9f0b9b1946"},"outputs":[{"output_type":"stream","name":"stdout","text":["Collecting cyvcf2\n","  Downloading cyvcf2-0.30.28-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (6.8 MB)\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m6.8/6.8 MB\u001b[0m \u001b[31m21.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hRequirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from cyvcf2) (1.25.2)\n","Collecting coloredlogs (from cyvcf2)\n","  Downloading coloredlogs-15.0.1-py2.py3-none-any.whl (46 kB)\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m46.0/46.0 kB\u001b[0m \u001b[31m5.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hRequirement already satisfied: click in /usr/local/lib/python3.10/dist-packages (from cyvcf2) (8.1.7)\n","Collecting humanfriendly>=9.1 (from coloredlogs->cyvcf2)\n","  Downloading humanfriendly-10.0-py2.py3-none-any.whl (86 kB)\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m86.8/86.8 kB\u001b[0m \u001b[31m10.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hInstalling collected packages: humanfriendly, coloredlogs, cyvcf2\n","Successfully installed coloredlogs-15.0.1 cyvcf2-0.30.28 humanfriendly-10.0\n"]}],"source":["import tensorflow as tf\n","from keras import regularizers\n","from keras.layers import Input, Dense, Dropout, Activation, Conv1D\n","from keras.layers import GlobalAveragePooling1D, AveragePooling1D\n","from keras.layers import Bidirectional, LSTM\n","from keras import Model\n","from keras.metrics import MeanSquaredError\n","\n","import pandas as pd\n","import numpy as np\n","\n","import requests\n","from functools import reduce\n","from operator import add\n","import tabulate\n","from difflib import SequenceMatcher\n","\n","!pip install cyvcf2\n","import cyvcf2\n","!pip install parasail\n","import parasail\n","\n","import re"]},{"cell_type":"markdown","source":["### One-hot encoding"],"metadata":{"id":"tt7r3eBnRrkF"}},{"cell_type":"code","execution_count":null,"metadata":{"id":"leFOTHSsl3VH"},"outputs":[],"source":["ntmap = {'A': (1, 0, 0, 0),\n","         'C': (0, 1, 0, 0),\n","         'G': (0, 0, 1, 0),\n","         'T': (0, 0, 0, 1)\n","         }\n","\n","def get_seqcode(seq):\n","    return np.array(reduce(add, map(lambda c: ntmap[c], seq.upper()))).reshape((1, len(seq), -1))"]},{"cell_type":"markdown","source":["### Crispr_BiLSTM model"],"metadata":{"id":"b6VehJkISeSw"}},{"cell_type":"code","execution_count":null,"metadata":{"id":"egLwUVQNTHIq"},"outputs":[],"source":["def BiLSTM_model(input_shape):\n","    input = Input(shape=input_shape)\n","\n","    conv1 = Conv1D(128, 5, activation=\"relu\")(input)\n","    pool1 = AveragePooling1D(2)(conv1)\n","    drop1 = Dropout(0.1)(pool1)\n","\n","    conv2 = Conv1D(128, 5, activation=\"relu\")(drop1)\n","    pool2 = AveragePooling1D(2)(conv2)\n","    drop2 = Dropout(0.1)(pool2)\n","\n","    lstm1 = Bidirectional(LSTM(128,\n","                               dropout=0.1,\n","                               activation='tanh',\n","                               return_sequences=True,\n","                               kernel_regularizer=regularizers.l2(1e-4)))(drop2)\n","    avgpool = GlobalAveragePooling1D()(lstm1)\n","\n","    dense1 = Dense(128,\n","                   kernel_regularizer=regularizers.l2(1e-4),\n","                   bias_regularizer=regularizers.l2(1e-4),\n","                   activation=\"relu\")(avgpool)\n","    drop3 = Dropout(0.1)(dense1)\n","\n","    dense2 = Dense(32,\n","                   kernel_regularizer=regularizers.l2(1e-4),\n","                   bias_regularizer=regularizers.l2(1e-4),\n","                   activation=\"relu\")(drop3)\n","    drop4 = Dropout(0.1)(dense2)\n","\n","    dense3 = Dense(32,\n","                   kernel_regularizer=regularizers.l2(1e-4),\n","                   bias_regularizer=regularizers.l2(1e-4),\n","                   activation=\"relu\")(drop4)\n","    drop5 = Dropout(0.1)(dense3)\n","\n","    output = Dense(1, activation=\"linear\")(drop5)\n","\n","    model = Model(inputs=[input], outputs=[output])\n","    return model\n"]},{"cell_type":"markdown","source":["### Predict gRNA in one specific gene"],"metadata":{"id":"7DJM4rUoUXmV"}},{"cell_type":"code","execution_count":null,"metadata":{"id":"FGTsGD31THLc"},"outputs":[],"source":["def fetch_ensembl_transcripts(gene_symbol):\n","    url = f\"https://rest.ensembl.org/lookup/symbol/homo_sapiens/{gene_symbol}?expand=1;content-type=application/json\"\n","    response = requests.get(url)\n","    if response.status_code == 200:\n","        gene_data = response.json()\n","        if 'Transcript' in gene_data:\n","            return gene_data['Transcript']\n","        else:\n","            print(\"No transcripts found for gene:\", gene_symbol)\n","            return None\n","    else:\n","        print(f\"Error fetching gene data from Ensembl: {response.text}\")\n","        return None\n","\n","def fetch_ensembl_sequence(transcript_id):\n","    url = f\"https://rest.ensembl.org/sequence/id/{transcript_id}?content-type=application/json\"\n","    response = requests.get(url)\n","    if response.status_code == 200:\n","        sequence_data = response.json()\n","        if 'seq' in sequence_data:\n","            return sequence_data['seq']\n","        else:\n","            print(\"No sequence found for transcript:\", transcript_id)\n","            return None\n","    else:\n","        print(f\"Error fetching sequence data from Ensembl: {response.text}\")\n","        return None\n"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"Rnkk3FHeTHOi"},"outputs":[],"source":["def find_crispr_targets(sequence, chr, start, end, strand, transcript_id, exon_id, pam=\"TTTN\", target_length=34):\n","    targets = []\n","    len_sequence = len(sequence)\n","    #complement = {'A': 'T', 'T': 'A', 'C': 'G', 'G': 'C'}\n","    dnatorna = {'A': 'A', 'T': 'U', 'C': 'C', 'G': 'G'}\n","\n","    for i in range(len_sequence - target_length + 1):\n","        target_seq = sequence[i:i + target_length]\n","        if target_seq[4:7] == 'TTT':\n","            if strand == -1:\n","                tar_start = end - i - target_length + 1\n","                tar_end = end -i\n","                #seq_in_ref = ''.join([complement[base] for base in target_seq])[::-1]\n","            else:\n","                tar_start = start + i\n","                tar_end = start + i + target_length - 1\n","                #seq_in_ref = target_seq\n","            gRNA = ''.join([dnatorna[base] for base in target_seq[8:28]])\n","            targets.append([target_seq, gRNA, chr, str(tar_start), str(tar_end), str(strand), transcript_id, exon_id])\n","            #targets.append([target_seq, gRNA, chr, str(tar_start), str(tar_end), str(strand), transcript_id, exon_id, seq_in_ref])\n","    return targets\n"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"lxTDh9nnTHZG"},"outputs":[],"source":["def format_prediction_output(targets, model_path):\n","    # Loading weights for the model\n","    Crispr_BiLSTM = BiLSTM_model(input_shape=(34, 4))\n","    Crispr_BiLSTM.load_weights(model_path)\n","\n","    formatted_data = []\n","    for target in targets:\n","        # Predict\n","        encoded_seq = get_seqcode(target[0])\n","        prediction = float(list(Crispr_BiLSTM.predict(encoded_seq, verbose=0)[0])[0])\n","        if prediction > 100:\n","            prediction = 100\n","\n","        # Format output\n","        gRNA = target[1]\n","        chr = target[2]\n","        start = target[3]\n","        end = target[4]\n","        strand = target[5]\n","        transcript_id = target[6]\n","        exon_id = target[7]\n","        #seq_in_ref = target[8]\n","        #formatted_data.append([chr, start, end, strand, transcript_id, exon_id, target[0], gRNA, seq_in_ref, prediction])\n","        formatted_data.append([chr, start, end, strand, transcript_id, exon_id, target[0], gRNA, prediction])\n","\n","    return formatted_data"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"mMU694bWXhbc"},"outputs":[],"source":["def gRNADesign(gene_symbol, model_path, write_to_csv=False):\n","    transcripts = fetch_ensembl_transcripts(gene_symbol)\n","    results = []\n","    if transcripts:\n","        for i in range(len(transcripts)):\n","            Exons = transcripts[i]['Exon']\n","            transcript_id = transcripts[i]['id']\n","            for j in range(len(Exons)):\n","                exon_id = Exons[j]['id']\n","                gene_sequence = fetch_ensembl_sequence(exon_id)\n","                if gene_sequence:\n","                    start = Exons[j]['start']\n","                    end = Exons[j]['end']\n","                    strand = Exons[j]['strand']\n","                    chr = Exons[j]['seq_region_name']\n","                    targets= find_crispr_targets(gene_sequence, chr, start, end, strand, transcript_id, exon_id)\n","                    if targets:\n","                        formatted_data = format_prediction_output(targets, model_path)\n","                        results.append(formatted_data)\n","\n","    #header = ['Chr','Start','End','Strand','Transcript','Exon','Target sequence (5\\' to 3\\')','gRNA','Sequence in reference genome','pred_Score']\n","    header = ['Chr','Start','End','Strand','Transcript','Exon','Target sequence (5\\' to 3\\')','gRNA','pred_Score']\n","\n","    output = []\n","    for result in results:\n","        for item in result:\n","            output.append(item)\n","    #sort_output = sorted(output, key=lambda x: x[9], reverse=True)\n","    sort_output = sorted(output, key=lambda x: x[8], reverse=True)\n","\n","    if write_to_csv==True:\n","        pd.DataFrame(data=sort_output, columns=header).to_csv(f'/content/drive/MyDrive/Colab Notebooks/Cas12/design_results/Cas12_{gene_symbol}.csv')\n","    else:\n","        return sort_output"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"i_UkwcHJXhiV"},"outputs":[],"source":["# example\n","model_path = '/content/drive/MyDrive/Colab Notebooks/Cas12/BiLSTM_Cpf1_weights.keras'\n","genes = ['TROAP','SPC24','RAD54L','MCM2','COPB2','CKAP5']\n","\n","for gene in genes:\n","    gRNADesign(gene, model_path, write_to_csv=False)"]},{"cell_type":"markdown","source":["### Predict cell line-specific gRNA"],"metadata":{"id":"FJprN1TcimaI"}},{"cell_type":"code","source":["def fetch_ensembl_transcripts(gene_symbol):\n","    url = f\"https://rest.ensembl.org/lookup/symbol/homo_sapiens/{gene_symbol}?expand=1;content-type=application/json\"\n","    response = requests.get(url)\n","    if response.status_code == 200:\n","        gene_data = response.json()\n","        if 'Transcript' in gene_data:\n","            return gene_data['Transcript']\n","        else:\n","            print(\"No transcripts found for gene:\", gene_symbol)\n","            return None\n","    else:\n","        print(f\"Error fetching gene data from Ensembl: {response.text}\")\n","        return None\n","\n","def fetch_ensembl_sequence(transcript_id):\n","    url = f\"https://rest.ensembl.org/sequence/id/{transcript_id}?content-type=application/json\"\n","    response = requests.get(url)\n","    if response.status_code == 200:\n","        sequence_data = response.json()\n","        if 'seq' in sequence_data:\n","            return sequence_data['seq']\n","        else:\n","            print(\"No sequence found for transcript:\", transcript_id)\n","            return None\n","    else:\n","        print(f\"Error fetching sequence data from Ensembl: {response.text}\")\n","        return None\n"],"metadata":{"id":"l1om4mF3qb8f"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["def apply_mutation(ref_sequence, offset, ref, alt):\n","    \"\"\"\n","    Apply a single mutation to the sequence.\n","    \"\"\"\n","    if len(ref) == len(alt) and alt != \"*\":  # SNP\n","        mutated_seq = ref_sequence[:offset] + alt + ref_sequence[offset+len(alt):]\n","\n","    elif len(ref) < len(alt):  # Insertion\n","        mutated_seq = ref_sequence[:offset] + alt + ref_sequence[offset+1:]\n","\n","    elif len(ref) == len(alt) and alt == \"*\":  # Deletion\n","        mutated_seq = ref_sequence[:offset] + ref_sequence[offset+1:]\n","\n","    elif len(ref) > len(alt) and alt != \"*\":  # Deletion\n","        mutated_seq = ref_sequence[:offset] + alt + ref_sequence[offset+len(ref):]\n","\n","    elif len(ref) > len(alt) and alt == \"*\":  # Deletion\n","        mutated_seq = ref_sequence[:offset] + ref_sequence[offset+len(ref):]\n","\n","    return mutated_seq\n","\n","\n","def construct_combinations(sequence, mutations):\n","    \"\"\"\n","    Construct all combinations of mutations.\n","    mutations is a list of tuples (position, ref, [alts])\n","    \"\"\"\n","    if not mutations:\n","        return [sequence]\n","\n","    # Take the first mutation and recursively construct combinations for the rest\n","    first_mutation = mutations[0]\n","    rest_mutations = mutations[1:]\n","    offset, ref, alts = first_mutation\n","\n","    sequences = []\n","    for alt in alts:\n","        mutated_sequence = apply_mutation(sequence, offset, ref, alt)\n","        sequences.extend(construct_combinations(mutated_sequence, rest_mutations))\n","\n","    return sequences"],"metadata":{"id":"y_h2tIrEUK8_"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["def needleman_wunsch_alignment(query_seq, ref_seq):\n","    \"\"\"\n","    Use Needleman-Wunsch alignment to find the maximum alignment position in ref_seq\n","    Use this position to represent the position of target sequence with mutations\n","    \"\"\"\n","    # Needleman-Wunsch alignment\n","    alignment = parasail.nw_trace(query_seq, ref_seq, 10, 1, parasail.blosum62)\n","\n","    # extract CIGAR object\n","    cigar = alignment.cigar\n","    cigar_string = cigar.decode.decode(\"utf-8\")\n","\n","    # record ref_pos\n","    ref_pos = 0\n","\n","    matches = re.findall(r'(\\d+)([MIDNSHP=X])', cigar_string)\n","    max_num_before_equal = 0\n","    max_equal_index = -1\n","    total_before_max_equal = 0\n","\n","    for i, (num_str, op) in enumerate(matches):\n","        num = int(num_str)\n","        if op == '=':\n","            if num > max_num_before_equal:\n","                max_num_before_equal = num\n","                max_equal_index = i\n","    total_before_max_equal = sum(int(matches[j][0]) for j in range(max_equal_index))\n","\n","    ref_pos = total_before_max_equal\n","\n","    return ref_pos\n"],"metadata":{"id":"4UiZvUWBIgIi"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["def find_gRNA_with_mutation(ref_sequence, exon_chr, start, end, strand, transcript_id,\n","                            exon_id, gene_symbol, vcf_reader, pam=\"TTTN\", target_length=34):\n","    # initialization\n","    mutated_sequences = [ref_sequence]\n","\n","    # find mutations within interested region\n","    mutations = vcf_reader(f\"{exon_chr}:{start}-{end}\")\n","    if mutations:\n","        # find mutations\n","        mutation_list = []\n","        for mutation in mutations:\n","            offset = mutation.POS - start\n","            ref = mutation.REF\n","            alts = mutation.ALT[:-1]\n","            mutation_list.append((offset, ref, alts))\n","\n","        # replace reference sequence of mutation\n","        mutated_sequences = construct_combinations(ref_sequence, mutation_list)\n","\n","    # find gRNA in ref_sequence or all mutated_sequences\n","    targets = []\n","    for seq in mutated_sequences:\n","        len_sequence = len(seq)\n","        dnatorna = {'A': 'A', 'T': 'U', 'C': 'C', 'G': 'G'}\n","        for i in range(len_sequence - target_length + 1):\n","            target_seq = seq[i:i + target_length]\n","            if target_seq[4:7] == 'TTT':\n","                pos = ref_sequence.find(target_seq)\n","                if pos != -1:\n","                    is_mut = False\n","                    if strand == -1:\n","                        tar_start = end - pos - target_length + 1\n","                    else:\n","                        tar_start = start + pos\n","                else:\n","                    is_mut = True\n","                    nw_pos = needleman_wunsch_alignment(target_seq, ref_sequence)\n","                    if strand == -1:\n","                        tar_start = str(end - nw_pos - target_length + 1) + '*'\n","                    else:\n","                        tar_start = str(start + nw_pos) + '*'\n","                gRNA = ''.join([dnatorna[base] for base in target_seq[8:28]])\n","                targets.append([target_seq, gRNA, exon_chr, str(strand), str(tar_start), transcript_id, exon_id, gene_symbol, is_mut])\n","\n","    # filter duplicated targets\n","    unique_targets_set = set(tuple(element) for element in targets)\n","    unique_targets = [list(element) for element in unique_targets_set]\n","\n","    return unique_targets"],"metadata":{"id":"fKHJQqY4ULAC"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["def format_prediction_output_with_mutation(targets, model_path):\n","    Crispr_BiLSTM = BiLSTM_model(input_shape=(34, 4))\n","    Crispr_BiLSTM.load_weights(model_path)\n","\n","    formatted_data = []\n","    for target in targets:\n","        # Predict\n","        encoded_seq = get_seqcode(target[0])\n","        prediction = float(list(Crispr_BiLSTM.predict(encoded_seq, verbose=0)[0])[0])\n","        if prediction > 100:\n","            prediction = 100\n","\n","        # Format output\n","        gRNA = target[1]\n","        exon_chr = target[2]\n","        strand = target[3]\n","        tar_start = target[4]\n","        transcript_id = target[5]\n","        exon_id = target[6]\n","        gene_symbol = target[7]\n","        is_mut = target[8]\n","        formatted_data.append([gene_symbol, exon_chr, strand, tar_start, transcript_id, exon_id, target[0], gRNA, prediction, is_mut])\n","\n","    return formatted_data"],"metadata":{"id":"M6cnzv1RULCh"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["def gRNADesign_mutation(gene_symbol, vcf_reader, model_path, write_to_csv=False):\n","    results = []\n","\n","    transcripts = fetch_ensembl_transcripts(gene_symbol)\n","    if transcripts:\n","        for transcript in transcripts:\n","            Exons = transcript['Exon']\n","            transcript_id = transcript['id']\n","\n","            for Exon in Exons:\n","                exon_id = Exon['id']\n","                exon_chr = Exon['seq_region_name']\n","                start = Exon['start']\n","                end = Exon['end']\n","                strand = Exon['strand']\n","                gene_sequence = fetch_ensembl_sequence(exon_id) # reference exon sequence\n","\n","                if gene_sequence:\n","                    targets = find_gRNA_with_mutation(gene_sequence, exon_chr, start, end, strand, transcript_id, exon_id, gene_symbol, vcf_reader)\n","                    if targets:\n","                        # Predict on-target efficiency for each gRNA site\n","                        formatted_data = format_prediction_output_with_mutation(targets, model_path)\n","                        results.append(formatted_data)\n","\n","    header = ['Gene','Chrom','Strand','Start','Transcript','Exon','Target sequence (5\\' to 3\\')','gRNA','pred_Score','Is_mutation']\n","\n","    output = []\n","    for result in results:\n","      for item in result:\n","        output.append(item)\n","    sort_output = sorted(output, key=lambda x: x[8], reverse=True)\n","\n","    if write_to_csv==True:\n","        pd.DataFrame(data=sort_output, columns=header).to_csv(f'/content/drive/MyDrive/Colab Notebooks/Cas12/design_results/Cas12_{gene_symbol}_mut.csv')\n","    else:\n","        return sort_output"],"metadata":{"id":"y63FEkionnyg"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# example\n","vcf_reader = cyvcf2.VCF('/content/drive/MyDrive/Colab Notebooks/CRISPR_data/SRR25934512.filter.snps.indels.vcf.gz')\n","model_path = '/content/drive/MyDrive/Colab Notebooks/Cas12/BiLSTM_Cpf1_weights.keras'\n","genes = ['TROAP','SPC24','RAD54L','MCM2','COPB2','CKAP5']\n","\n","for gene in genes:\n","    gRNADesign_mutation(gene, vcf_reader, model_path, write_to_csv=False)"],"metadata":{"id":"t7tsjoR3UK6W"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":[],"metadata":{"id":"_zQi-1e3X3Kx"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":[],"metadata":{"id":"KLW_WGaRX3NS"},"execution_count":null,"outputs":[]}],"metadata":{"colab":{"provenance":[],"toc_visible":true,"authorship_tag":"ABX9TyPFLC90NR314R1kFJp9HLQu"},"kernelspec":{"display_name":"Python 3","name":"python3"},"language_info":{"name":"python"}},"nbformat":4,"nbformat_minor":0}