import json
import matplotlib.pyplot as plt
def read_corpus():
    """
    读取给定的语料库，并把问题列表和答案列表分别写入到 qlist, alist 里面。 在此过程中，不用对字符换做任何的处理（这部分需要在 Part 2.3里处理）
    qlist = ["问题1"， “问题2”， “问题3” ....]
    alist = ["答案1", "答案2", "答案3" ....]
    务必要让每一个问题和答案对应起来（下标位置一致）
    """
    # TODO 需要完成的代码部分 ...
    filename="./train-v2.0.json"
    qlist=[]
    alist=[]
    with open(filename) as f:
        data_array = json.load(f)['data']
        for data in data_array:
            paragraphs = data['paragraphs']
            for paragraph in paragraphs:
                qas = paragraph['qas']
                for qa in qas:
                    for qa in qas:
                        if 'plausible_answers' in qa:
                            qlist.append(qa['question'])
                            alist.append(qa['plausible_answers'][0]['text'])
                        else:
                            qlist.append(qa['question'])
                            alist.append(qa['answers'][0]['text'])


    with open('a_list.json','w') as file:
        json.dump(alist,file)
    assert len(qlist) == len(alist)  # 确保长度一样
    return qlist, alist

# TODO: 统计一下在qlist中总共出现了多少个单词？ 总共出现了多少个不同的单词(unique word)？
#       这里需要做简单的分词，对于英文我们根据空格来分词即可，其他过滤暂不考虑（只需分词）
def count_word(qlist):
    qlist_dic={}
    word_total=0

    for line in qlist:
        words=line.strip().split()
        for item in words:
            word_total+=1
            if item in qlist_dic:
                qlist_dic[item]+=1
            else:
                qlist_dic[item]=1
    return qlist_dic,word_total
qlist,alist=read_corpus()
qlist_dic,word_total=count_word(qlist)
print (word_total,len(qlist_dic.keys()))
    

# TODO: 统计一下qlist中出现1次，2次，3次... 出现的单词个数， 然后画一个plot. 这里的x轴是单词出现的次数（1，2，3，..)， y轴是单词个数。
#       从左到右分别是 出现1次的单词数，出现2次的单词数，出现3次的单词数... 
from collections import Counter
def plot_table(qlist_dic):
    words_freq=qlist_dic.values()
    words_freq_dic=Counter(words_freq)  #此时字典中的键就是词频，值就是具有相同词频的单词数量  
    x=sorted(words_freq_dic.keys()) #排序，确保词频，即X轴上递增
    y=[]
    #找到每个词频对应的单词数量
    for i in x:
        y.append(words_freq_dic[i])
    
    plt.plot(x, y, color="r", linestyle="-", marker="*", linewidth=1.0)
    plt.show()
plot_table(qlist_dic) 

# TODO： 从上面的图中能观察到什么样的现象？ 这样的一个图的形状跟一个非常著名的函数形状很类似，能所出此定理吗？ 
#     answer:  [Zip]'s law吧，就是低频的词占了绝大多数，高频词其实很少
#       就是不知道为啥叫Zip 
# 

# TODO： 需要做文本方面的处理。 从上述几个常用的方法中选择合适的方法给qlist做预处理（不一定要按照上面的顺序，不一定要全部使用）
from nltk.corpus import stopwords
import string
#利用上面得到的有序词频列表x，选取百分比来确定低频值
def get_lowfreq_list(qlist_dic,percent):
    words_freq=qlist_dic.values()
    words_freq_dic=Counter(words_freq)  #此时字典中的键就是词频，值就是具有相同词频的单词数量  
    x=sorted(words_freq_dic.keys())
    
    index=float(percent)*len(x)
    index=int(index)+1#避免取0值
    low_freq=x[index]
    low_freq_list=[]
    for i,j in qlist_dic.items():
        if j<low_freq:
            low_freq_list.append(i)
    return low_freq_list
#不替换数字，也不去除低频词
def text_preprocess(input_list):
    #low_freq_list=get_lowfreq_list(qlist_dic,percent)
    stop_words = stopwords.words('english')
    new_list=[]
    for line in input_list:
        line=line.lower()
        line=''.join(c for c in line if c not in string.punctuation)#去除无用符号
        split_line=line.strip().split()
        #print(words)
        new_line=[word for word in split_line if word not in stop_words]
        filtered_line=' '.join(new_line)
        new_list.append(filtered_line)
    print(new_list[0])
    with open("qlist.json",'w')as file:
        json.dump(new_list,file)
    return new_list

qlist,alist=read_corpus()
new_list= text_preprocess(qlist)    # 更新后的问题列表

# TODO 
from sklearn.feature_extraction.text import TfidfVectorizer
def Tfidf_vec(qlist):
    vectorizer = TfidfVectorizer()# 定义一个tf-idf的vectorizer
    X = vectorizer.fit_transform(qlist)  
    # 结果存放在X矩阵里
    return X


#print(Tfidf_vec(qlist)[0])

# TODO 基于Glove向量获取句子向量
def get_glove_vec(qlist):
    import numpy as np
    with open("glove.6B.200d.txt","r",encoding="utf-8") as glovefile:
        word2vec={}
        for i,each_line in enumerate(glovefile):
            each_vec=each_line.strip().split(' ')
            word2vec[each_vec[0]]=[float(i) for i in each_vec[1:]]
    with open("glove.json",'w')as file:
        json.dump(word2vec,file)
        
    print(word2vec['of'])
    X_w2v=[]    
    for line in qlist:
        words=line.strip().split()
        line_vec=[]
        count_word=0
        for word in words:
            if word in word2vec: 
                count_word+=1
                line_vec.append(word2vec[word])
        X_w2v.append(np.sum(np.array(line_vec),axis=0) /count_word)  
        #print(X_w2v)
             # 初始化完emb之后就可以对每一个句子来构建句子向量了，这个过程使用average pooling来实现
    X_w2v=np.asarray(X_w2v)
    return X_w2v
get_glove_vec(qlist)

# TODO 基于BERT的句子向量计算
!pip install bert_embedding
def get_bert_vec(qlist):
    import numpy as np
    from bert_embedding import BertEmbedding
    bert_embedding = BertEmbedding(model='bert_12_768_12', dataset_name='book_corpus_wiki_en_cased')
    bert=[]
    for line in qlist:
        result = bert_embedding([line])
        item=result[0]
        if len(item[0])>0:
            bert.append(np.sum(np.array(item[1]),axis=0)/len(item[0]))
    X_bert =np.asarray(bert)   # 每一个句子的向量结果存放在X_bert矩阵里。行数为句子的总个数，列数为一个句子embedding大小。
    print(X_bert[0])
    return X_bert
get_bert_vec(qlist)

import heapq as hp
from sklearn.metrics.pairwise import cosine_similarity
#处理查询句子
def movestopwords(sentence):
    from nltk.corpus import stopwords
    # 英文的大写全部改成小写
    q_list_lower = sentence.lower()    
    # 把问题拆解成一个一个单词
    word_tokens =q_list_lower.strip().split()         
    # 过滤掉停用词
    ## 增加stopwords
    stop_words = set(stopwords.words('english'))
    stop_words.update(['.', ',', '"', "'", '?', '!', ':', ';', '(', ')', '[', ']', '{', '}'])
    filtered_words = [word for word in word_tokens if word not in stop_words]
    sent=" ".join(filtered_words)
    return sent
def get_top_results_tfidf_noindex(query):
    # TODO 需要编写
    """
    给定用户输入的问题 query, 返回最有可能的TOP 5问题。这里面需要做到以下几点：
    1. 对于用户的输入 query 首先做一系列的预处理(上面提到的方法)，然后再转换成tf-idf向量（利用上面的vectorizer)
    2. 计算跟每个库里的问题之间的相似度
    3. 找出相似度最高的top5问题的答案
    """
    import heapq 
    with open('qlist.json','r')as file:
        q_list=file.read()
        qlist=json.loads(q_list)
    vectorizer = TfidfVectorizer()
    qlist_vec=vectorizer.fit_transform(qlist)
    #print(qlist_vec[0])
    sent=movestopwords(query)
    query_vec=vectorizer.transform([sent])
    #print(query_vec[0])
    #计算相似度最高的前5个
    
    all_result=cosine_similarity(qlist_vec,query_vec)[0]
    print(all_result)
    cos_list=list(all_result[0])
    top_idxs =map(cos_list.index, heapq.nlargest(5, cos_list)) 
    #print(top_idxs)
    top_answer= []  # top_idxs存放相似度最高的（存在qlist里的）问题的下标 
                   # hint: 请使用 priority queue来找出top results. 思考为什么可以这么做？
    with open('a_list.json','r')as file:
        a_list=file.read()
        alist=json.loads(a_list)
    for index in top_idxs:
        top_answer.append(alist[index])
    return top_answer  # 返回相似度最高的问题对应的答案，作为TOP5答案    

# TODO: 编写几个测试用例，并输出结果
sent1=movestopwords("when did Beyonce start singing")
print (get_top_results_tfidf_noindex(sent1))
sent2=movestopwords("what is machine learning")
print (get_top_results_tfidf_noindex(sent2))

def create_inverted_index(qlist_dic):
    word_total = [item.lower() for item in qlist_dic.keys()]
    inverted_idx = {}  # 定一个一个简单的倒排表，是一个map结构。 循环所有qlist一遍就可以
    with open('q_list.json') as file:
        q_list=file.read()
        qlist=json.loads(q_list)

    for index,i in enumerate(word_total):
        tmp=[]
        j=0
        while j<len(qlist):
            field=qlist[j]
            split_field=field.split()
            if i in split_field:
                tmp.append(j)
            j+=1
        inverted_idx[i]=tmp
            
    with open('inverted_idx.json','w') as file_object:
        json.dump(inverted_idx,file_object)
    return inverted_idx

# TODO 读取语义相关的单词
def get_related_words(file):
    
    return related_words

related_words = get_related_words('related_words.txt') # 直接放在文件夹的根目录下，不要修改此路径。

#第一大模块
#处理查询句子
def q_movestopwords(sentence):
    from nltk.corpus import stopwords
    # 英文的大写全部改成小写
    q_list_lower = sentence.lower()    

    # 把问题拆解成一个一个单词
    word_tokens =q_list_lower.strip().split()
    # 过滤掉停用词
    ## 增加stopwords
    stop_words = set(stopwords.words('english'))
    stop_words.update(['.', ',', '"', "'", '?', '!', ':', ';', '(', ')', '[', ']', '{', '}'])
    filtered_words = [word for word in word_tokens if word not in stop_words]
    filtered_sent=" ".join(filtered_words)

    return filtered_words,filtered_sent



#第二大模块
#取得查询语句意义相近的词
def get_related_words(filtered_list):
    import json
    # 读取word的相似词字典.json
    with open('related_words.json', 'r') as fp:
        related_words_dic = json.load(fp)
        
    simi_list = []
    for w in filtered_list:
        temp = related_words_dic[w]
        temp_list = temp[0]
        simi_list += temp_list
    return simi_list

#取得相近的词在整个问题列表中的索引
def get_inverted_idx(simi_list):
    import json
    # 读取word的倒排表字典.json
    with open('inverted_idx.json', 'r') as fp:
        inverted_idx_dic = json.load(fp)
    
    # 查找问题的index列表
    question_list_idnex = []
    for word in simi_list:
    
        # 只找字典中有的单词进行查找
        if word in inverted_idx_dic:
            question_list_idnex += inverted_idx_dic[word]

    # 清除重复的问题的index        
    index_q = list(set(question_list_idnex))
    #print(index_q)
    return index_q

#找到问题与问题集相似的所有问题
def get_total_q(index_q):
    import json
    with open('q_list.json','r')as file:
        tem_qlist=file.read()
        qlist=json.loads(tem_qlist)
        #print(qlist.shape)
    q_total_list = []
    for q in index_q:
        q_total_list.append(qlist[q])
    #print(q_total_list)
    return q_total_list

 
#找出前k个余弦相似度最高的问题
def get_cos_list(query,q_total_list,k):
    import heapq
    from sklearn.feature_extraction.text import TfidfVectorizer
    from sklearn.metrics.pairwise import cosine_similarity
    vectorizer = TfidfVectorizer()
    #query_list=query.strip().split()
    q_list_tfidf=vectorizer.fit_transform(q_total_list)
    sent_tfidf=vectorizer.transform([query])
    all_result=cosine_similarity(sent_tfidf,q_list_tfidf)
    #排序并记录最相似问题的索引
    cos_list=list(all_result[0])
    max_index =map(cos_list.index, heapq.nlargest(k, cos_list)) 
    return max_index
#找出最相似的问题    
def get_top_answer(max_index,q_total_clean_list):
    import json
    with open('a_list.json', 'r') as file:
        answers = file.read()
        alist = json.loads(answers)
    #由于记录最相似词在问题列表中的位置的索引表和问题的句子向量一一对应，故通过后者在列表中位置问题qlist的索引
    top_index=[]
    for i in max_index:
        if i <len(q_total_clean_list):
            top_index.append(q_total_clean_list[i])
     
    top_answer=[]    
    for i in top_index:
        top_answer.append([alist[i]])
    return top_answer
        
#合并所有流程
def get_top_results_tfidf(query):
    import time
    # 找出与问题与问题集中相似的问题
    filtered_words,filtered_sent=q_movestopwords(query)
    simi_list=get_related_words(filtered_words)
    q_total_index=get_inverted_idx(simi_list)
    q_total_list=get_total_q(q_total_index)   
 
    #取得新问题对相关问题的相似度
    start_time=time.time()
    max_index = get_cos_list(filtered_sent,q_total_list,5)
    end_time=time.time()
    print(end_time-start_time)
    #取得最接近的答案list
    top_answer=get_top_answer(max_index,q_total_index)
    return top_answer

#利用前面定义好的将文档转为glove向量的函数，query的glove表示单独处理
def get_w2v_list(filtered_sent,q_total_list,k):
    import heapq
    import numpy as np
    q_list_w2v=get_glove_vec(q_total_list)
    with open("glove.json",'r')as file:
        glove=file.read()
        glove_dic=json.loads(glove)
    #获取query的glove向量表示
    query=filtered_sent.strip().split()
    query_len=len(query)
    query_vec=[]
    for word in query:
        if word in glove_dic: 
            query_vec.append(glove_dic[word])
    final_vec=np.sum(np.array(query_vec))/query_len    
    all_result=cosine_similarity(final_vec,q_list_w2v)
    #排序并记录最相似问题的索引
    cos_list=list(all_result[0])
    max_index =map(cos_list.index, heapq.nlargest(k, cos_list)) 
    return max_index
#复用已有函数，只改变获取文档向量表示的函数
def get_top_results_w2v(query):
    import time
    # 找出与问题与问题集中相似的问题
    filtered_words,filtered_sent=q_movestopwords(query)
    simi_list=get_related_words(filtered_words)
    q_total_index=get_inverted_idx(simi_list)
    q_total_list=get_total_q(q_total_index)   
 
    #取得新问题对相关问题的相似度
    start_time=time.time()
    max_index = get_w2v_list(filtered_sent,q_total_list,5)
    end_time=time.time()
    print(end_time-start_time)
    #取得最接近的答案list
    top_answer=get_top_answer(max_index,q_total_index)
    return top_answer
 

#利用前面定义好的将文档转为bert向量的函数，query的glove表示单独处理
def get_bert_list(filtered_sent,q_total_list,k):
    import heapq
    q_list_bert=get_bert_vec(q_total_list)
    import numpy as np
    from bert_embedding import BertEmbedding
    bert_embedding = BertEmbedding(model='bert_12_768_12', dataset_name='book_corpus_wiki_en_cased')
    
    query=filtered_sent.strip().split()
    result = bert_embedding(query)
    item=result[0]
    query_bert=np.sum(np.array(item[1],axis=0))/len(item[0])
    all_result=cosine_similarity(query_bert,q_list_bert)
    #排序并记录最相似问题的索引
    cos_list=list(all_result[0])
    max_index =map(cos_list.index, heapq.nlargest(k, cos_list)) 
    return max_index
#复用已有函数，只改变获取文档向量表示的函数
def get_top_results_bert(query):
    import time
    # 找出与问题与问题集中相似的问题
    filtered_words,filtered_sent=q_movestopwords(query)
    simi_list=get_related_words(filtered_words)
    q_total_index=get_inverted_idx(simi_list)
    q_total_list=get_total_q(q_total_index)   
 
    #取得新问题对相关问题的相似度
    start_time=time.time()
    max_index = get_bert_list(filtered_sent,q_total_list,5)
    end_time=time.time()
    print(end_time-start_time)
    #取得最接近的答案list
    top_answer=get_top_answer(max_index,q_total_index)
    return top_answer
 

# TODO: 编写几个测试用例，并输出结果

test_query1 = "when did Beyonce start singing"
test_query2 = "what is machine learning"

print (get_top_results_tfidf(test_query1))
print (get_top_results_w2v(test_query1))
print (get_top_results_bert(test_query1))

print (get_top_results_tfidf(test_query2))
print (get_top_results_w2v(test_query2))
print (get_top_results_bert(test_query2))

from nltk.corpus import reuters

# 读取语料库的数据
categories = reuters.categories()
corpus = reuters.sents(categories=categories)

# 文本整理成list和把所有不重复单词整理成list
all_word_list = []
all_word = []

for file in  reuters.fileids():
    file_word = reuters.words(file)
    all_word += list(file_word)
    all_word_list += [list(file_word)]
# 去除重复单词
total_word = list(set(all_word))


import json
# total_word和All_word资料整理成json
file_name_1 = 'all_word_list.json'
file_name_2 = 'total_word.json'
file_name_3 = 'all_word.json'

with open(file_name_1,'w') as file_object:
    json.dump(all_word_list, file_object)
with open(file_name_2,'w') as file_object:
    json.dump(total_word, file_object)
with open(file_name_3,'w') as file_object:
    json.dump(all_word, file_object) 
#计算bigram的几率值

# 循环所有的语料库并构建bigram probability. bigram[word1][word2]: 在word1出现的情况下下一个是word2的概率。 


def get_bigram_pro(word1,word2,all_word_list = None,total_word = None):
    if all_word_list == None:
        import json
        # 读取所有问题
        with open('all_word_list.json','r') as file:
            all_word_list = json.load(file)
    if total_word == None:
        import json
        # 读取所有不重复单字
        with open('total_word.json','r') as file:
            total_word = json.load(file)
    # 所有文檔内文字列表
    from nltk.util import ngrams
    text_bigrams = [ngrams(sent, 2) for sent in all_word_list]
    text_unigrams = [ngrams(sent, 1) for sent in all_word_list]
    
    # 计算数量
    from nltk.lm import NgramCounter
    ngram_counts = NgramCounter(text_bigrams + text_unigrams)  
    
    #计算几率用add-one smoothing
    word_count = ngram_counts[word1]
    join_count = ngram_counts[[word1]][word2]
    total_word_length = len(total_word)
    
    bigram_probability = (join_count + 1) / ( word_count + total_word_length)
    return bigram_probability

channel = {}

spell_error_dict = {}

for line in open('spell-errors.txt'):
    item = line.split(":")
    word = item[0].strip()
    spell_error_list = [word.strip( )for word in item[1].strip().split(",")]
    spell_error_dict[word] = spell_error_list
# print_format("spell_error_dict", spell_error_dict)

for key in spell_error_dict:
    if key not in channel:
        channel[key] = {}
        for value in spell_error_dict[key]:
            channel[key][value] = 1 / len(spell_error_dict[key])
print(channel['raining']['rainning'])

    #找出所有英文字
def word_lower(text):
    import re
    return re.findall('[a-z]+',text.lower())

def known(words):
    # 打开单词列表
    import json 
    with open('total_word.json','r') as file:
        words_N = json.load(file)    
    
    return set(w for w in words if w in words_N)
#获取编辑距离为1的单词
def edits1(word):
    # 替换的英文字
    alphabet = "abcdefghijklmnopqrstuvwxyz"
    
    n = len(word)
    #删除
    s1 = [word[0:i]+word[i+1:] for i in range(n)]
    #调换相邻的两个字母
    s2 = [word[0:i]+word[i+1]+word[i]+word[i+2:] for i in range(n-1)]
    #replace
    s3 = [word[0:i]+c+word[i+1:] for i in range(n) for c in alphabet]
    #插入
    s4 = [word[0:i]+c+word[i:] for i in range(n+1) for c in alphabet]
    edits1_words = set(s1+s2+s3+s4)
    edits1_words.remove(word)
    edits1_words = known(edits1_words)
    return edits1_words

#2
def edits2(word):
    edits2_words = set(e2 for e1 in edits1(word) for e2 in edits1(e1))
    #edits2_words.remove(word)
    edits2_words = known(edits2_words)
    return edits2_words

def generate_candidates(word):
    #大写变小写
    word_lower_clean = word_lower(word)
    word_clean = word_lower_clean[0]
    
    # 打开单词列表
    import json 
    with open('total_word.json','r') as file:
        words_N = json.load(file)
    
    # 纠错
    if word_clean not in words_N:
        candidates = edits1(word_clean) or edits2(word_clean)
        return candidates
    else:
        return None
    

def get_bigram_chnnel_pro(tokens , index, all_word_list, total_word):
    # 前后文的单字
    front_word = tokens[index-1]
    now_word = tokens[index]
    after_word = tokens[index+1]
    
    #前后文的几率
    front_pro = get_bigram_pro(now_word , front_word , all_word_list , total_word)
    after_pro = get_bigram_pro(now_word , after_word , all_word_list , total_word)
    
    #计算bigram几率
    bigram_pro = front_pro * after_pro
    
    return bigram_pro
def spell_corector(line):
    # 1. 首先做分词，然后把``line``表示成``tokens``
    # 2. 循环每一token, 然后判断是否存在词库里。如果不存在就意味着是拼写错误的，需要修正。 
    #    修正的过程就使用上述提到的``noisy channel model``, 然后从而找出最好的修正之后的结果。
    
    # 句子先tokens,数据清洗
    line_clean = line.replace('?','')
    tokens = line_clean.split()
    
    
    #读取要使用的单字清单
    import json
    # 读取所有不重复单字
    with open('all_word_list.json','r') as file:
        all_word_list = json.load(file)
    # 读取还有所有问题
    with open('total_word.json','r') as file:
        total_word = json.load(file)
    
    # 逐个单字检查
    t = 0
    sentence = []
    
    while t < len(tokens):
        if tokens[t] not in total_word:

            # 找出备选相似词
            simility_list = list(generate_candidates(tokens[t]))
            
            # 列出备选项的几率列表,并找出几率最高的单词
            simility_pro_list = [get_bigram_chnnel_pro(tokens , t , all_word_list, total_word) for i in simility_list]
            
            # 避免专有名词影响,若找不到相似词,当专有名词
            if simility_pro_list == []:
                sentence += [tokens[t]]
            else:
                word_index = simility_pro_list.index(max(simility_pro_list))
                correct_word = simility_list[word_index]
            
                # 组成list
                sentence += [correct_word]
        
        else:
            sentence += [tokens[t]]
    
        t += 1

        # 将单词组合成句子
        newline = ' '.join(sentence)
    
    
    return newline 

test_query1 = "when did beyonce staart singing"  # 拼写错误的
test_query2 = "what is machin learning"  # 拼写错误的

test_query1 = spell_corector(test_query1)
test_query2 = spell_corector(test_query2)

print (get_top_results_tfidf(test_query1))
print (get_top_results_w2v(test_query1))
print (get_top_results_bert(test_query1))

print (get_top_results_tfidf(test_query2))
print (get_top_results_w2v(test_query2))
print (get_top_results_bert(test_query2))