#导入数据处理的基础包
import numpy as np
import pandas as pd

#导入用于计数的包
from collections import Counter

#导入tf-idf相关的包
from sklearn.feature_extraction.text import TfidfTransformer    
from sklearn.feature_extraction.text import CountVectorizer

#导入模型评估的包
from sklearn import metrics

#导入与word2vec相关的包
from gensim.models import KeyedVectors

#导入与bert embedding相关的包，关于mxnet包下载的注意事项参考实验手册
from bert_embedding import BertEmbedding
import mxnet

#包tqdm是用来对可迭代对象执行时生成一个进度条用以监视程序运行过程
from tqdm import tqdm

#导入其他一些功能包
import requests
import os
import re
import warnings
warnings.filterwarnings("ignore")

# 由于bert嵌入无法使用GPU进行加速，所以这里使用了比例系数K来减小数据量
K = 5000

#读取数据
data = pd.read_csv('data/DMSC.csv')
#观察数据格式
data.head()

#输出数据的一些相关信息
data.info()

#只保留数据中我们需要的两列：Comment列和Star列
data = data[['Comment','Star']]
#观察新的数据的格式
data.head()

# 这里的star代表具体的评分。但在这个项目中，我们要预测的是正面还是负面。我们把评分为1和2的看作是负面，把评分为3，4，5的作为正面
data['Star']=(data.Star/3).astype(int)
data.head(20)

# TODO1: 去掉一些无用的字符，自行定一个字符几何，并从文本中去掉
def pre_process(input_str):
    # input_str = re.sub('[0-9]+', 'DIG', input_str)
    # 去除标点符号
    # input_str = re.sub(r"[{}]+".format(punc), " ", input_str)
    
    input_str = re.sub(
        "[0-9a-zA-Z\-\s+\.\!\/_,$%^*\(\)\+(+\"\')]+|[+——！，。？、~@#￥%……&*（）<>\[\]:：★◆【】《》;；=?？]+", " ", input_str)
    # 其他非中文字符
    input_str = re.sub(r"[^\u4e00-\u9fff]", " ", input_str)
    return input_str.strip()

# 正则去除标点符号
data['comment_clean'] = data['Comment'].apply(pre_process)
data.head(20)

# TODO2: 导入中文分词包jieba, 并用jieba对原始文本做分词
import jieba
def comment_cut(content):
    # TODO: 使用结巴完成对每一个comment的分词
    # 分词并过滤空字符串
    return ' '.join([w for w in jieba.lcut(content.strip()) if len(w) > 0])

# 输出进度条
tqdm.pandas(desc='apply')
data['comment_processed'] = data['comment_clean'].progress_apply(comment_cut)

# 观察新的数据的格式
data.head()

# TODO3: 设定停用词并从文本中去掉停用词

# 下载中文停用词表至data/stopWord.json中，下载地址:https://github.com/goto456/stopwords/
if not os.path.exists('./data/stopWord.json'):
    stopWord = requests.get("https://raw.githubusercontent.com/goto456/stopwords/master/cn_stopwords.txt")
    with open("./data/stopWord.json", "wb") as f:
         f.write(stopWord.content)

# 读取下载的停用词表，并保存在列表中
with open("./data/stopWord.json","r", encoding="utf-8") as f:
    stopWords = f.read().split("\n")  
    
    
# 去除停用词
def rm_stop_word(input_str):
    # your code, remove stop words
    # TODO
    return [w for w in input_str.split() if w not in stopWords]

#这行代码中.progress_apply()函数的作用等同于.apply()函数的作用，只是写成.progress_apply()函数才能被tqdm包监控从而输出进度条。
data['comment_processed'] = data['comment_processed'].progress_apply(rm_stop_word)

data.head(20)

# TODO4: 去除低频词, 去掉词频小于10的单词，并把结果存放在data['comment_processed']里
word_counter = Counter([w for s in data['comment_processed'].values for w in s])


def rm_low_frequency_words(word_list):
    return [w for w in word_list if word_counter[w] >= 10]

data['comment_processed'] = data['comment_processed'].progress_apply(rm_low_frequency_words)
data['comment_processed_str'] = data['comment_processed'].apply(lambda x: ' '.join(x))
data.head(10)

# TODO5: 把数据分为训练集和测试集. comments_train（list)保存用于训练的文本，comments_test(list)保存用于测试的文本。 y_train, y_test是对应的标签（0、1）

from sklearn.model_selection import train_test_split

test_ratio = 0.2

# https://machinelearningmastery.com/train-test-split-for-evaluating-machine-learning-algorithms/
src_training, src_testing = train_test_split(data, test_size=test_ratio, stratify=data['Star'])

comments_train, comments_test = src_training['comment_processed_str'].values, src_testing['comment_processed_str'].values
y_train, y_test = src_training['Star'].values, src_testing['Star'].values

# TODO6: 把训练文本和测试文本转换成tf-idf向量。使用sklearn的feature_extraction.text.TfidfTransformer模块
#    请留意fit_transform和transform之间的区别。 常见的错误是在训练集和测试集上都使用 fit_transform，需要避免！ 
#    另外，可以留意一下结果是否为稀疏矩阵

from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfTransformer

count_vectorizer = CountVectorizer(token_pattern=r"(?u)\b\w+\b")
tfidf_transformer = TfidfTransformer()

word_count_train = count_vectorizer.fit_transform(comments_train)
tfidf_train = tfidf_transformer.fit_transform(word_count_train)

word_count_test = count_vectorizer.transform(comments_test)
tfidf_test = tfidf_transformer.transform(word_count_test)

print(tfidf_train.shape, tfidf_test.shape)

# 由于训练出一个高效的word2vec词向量往往需要非常大的语料库与计算资源，所以我们通常不自己训练Wordvec词向量，而直接使用网上开源的已训练好的词向量。
# data/sgns.zhihu.word是从https://github.com/Embedding/Chinese-Word-Vectors下载到的预训练好的中文词向量文件
# 使用KeyedVectors.load_word2vec_format()函数加载预训练好的词向量文件
model = KeyedVectors.load_word2vec_format('data/sgns.zhihu.word')

#预训练词向量使用举例
model['我们']

vocabulary = model.vocab

def word_vec_averaging(words, dim=300):
    """
    Average all words vectors in one sentence.
    :param words: input sentence
    :param dim: 'size' of model
    :return: the averaged word vectors as the vector for the sentence
    """
    vec_mean = np.zeros((dim,), dtype=np.float32)
    word_num = 0
    first_dim_sum = 0
    for word in words:
        print(f'Word is: {word}')
        if word in vocabulary:
            word_num += 1
            vec_mean = np.add(vec_mean, model[word])
            first_dim_sum += model[word][0]
            print(f'in vocab with first dimension: {model[word][0]} and first_dim_sum is: {first_dim_sum}')
        else:
            print('not in vocab')
    if word_num > 0:
        vec_mean = np.divide(vec_mean, word_num)
        print(f'word_num is: {word_num}, first dim average is: {first_dim_sum / word_num}')
    return vec_mean

one_sample = comments_train[100]
word_vec_averaging(one_sample.split())

comments_train[3]

vocabulary = model.vocab

def word_vec_averaging(words, dim=300):
    """
    Average all words vectors in one sentence.
    :param words: input sentence
    :param dim: 'size' of model
    :return: the averaged word vectors as the vector for the sentence
    """
    vec_mean = np.zeros((dim,), dtype=np.float32)
    word_num = 0
    first_dim_sum = 0
    for word in words:
        if word in vocabulary:
            word_num += 1
            vec_mean = np.add(vec_mean, model[word])
            first_dim_sum += model[word][0]
    if word_num > 0:
        vec_mean = np.divide(vec_mean, word_num)
    return vec_mean

word2vec_train = np.array([word_vec_averaging(s.split()) for s in comments_train])
word2vec_test = np.array([word_vec_averaging(s.split()) for s in comments_test])
print(word2vec_train.shape, word2vec_test.shape)

# 导入gpu版本的bert embedding预训练的模型。
# 若没有gpu，则ctx可使用其默认值cpu(0)。但使用cpu会使程序运行的时间变得非常慢
# 若之前没有下载过bert embedding预训练的模型，执行此句时会花费一些时间来下载预训练的模型
ctx = mxnet.cpu()
embedding = BertEmbedding(ctx=ctx)

# TODO8: 跟word2vec一样，计算出训练文本和测试文本的向量，仍然采用单词向量的平均。
def bert_embedding_averaging(sentence):
    """返回sentence bert 句向量"""
    tokens, token_embeddings = embedding([sentence])[0]
    return np.mean(np.array(token_embeddings), axis=0).astype(np.float32)
bert_train = np.array([bert_embedding_averaging(s) for s in comments_train[:len(comments_train)//K]])
bert_test = np.array([bert_embedding_averaging(s) for s in comments_test[:len(comments_test)//K]])
print (bert_train.shape, bert_test.shape)

print (tfidf_train.shape, tfidf_test.shape)
print (word2vec_train.shape, word2vec_test.shape)
print (bert_train.shape, bert_test.shape)

# 导入逻辑回归的包
from sklearn.linear_model import LogisticRegression

# TODO9: 使用tf-idf + 逻辑回归训练模型，需要用gridsearchCV做交叉验证，并选择最好的超参数
clf = LogisticRegression()

from sklearn.model_selection import GridSearchCV

search_grid = {
    'C': [0.01, 1, 10, 100],
    'class_weight': [None, 'balanced']
}

grid_search = GridSearchCV(estimator = clf, 
                           param_grid = search_grid, 
                           cv = 55, 
                           n_jobs=-1, 
                           scoring='accuracy')

grid_result = grid_search.fit(tfidf_train, y_train)
print(f'Best parameters: {grid_result.best_params_}')

clf.fit(tfidf_train, y_train)
tf_idf_y_pred = clf.predict(tfidf_test)
print('TF-IDF LR test accuracy %s' % metrics.accuracy_score(y_test, tf_idf_y_pred))
#逻辑回归模型在测试集上的F1_Score
print('TF-IDF LR test F1_score %s' % metrics.f1_score(y_test, tf_idf_y_pred,average="macro"))

# TODO10: 使用word2vec + 逻辑回归训练模型，需要用gridsearchCV做交叉验证，并选择最好的超参数
clf = LogisticRegression()

from sklearn.model_selection import GridSearchCV

search_grid = {
    'C': [0.01, 1, 10, 100],
    'class_weight': [None, 'balanced']
}

grid_search = GridSearchCV(estimator = clf, 
                           param_grid = search_grid, 
                           cv = 5, 
                           n_jobs=-1, 
                           scoring='accuracy')

grid_result = grid_search.fit(word2vec_train, y_train)
print(f'Best parameters: {grid_result.best_params_}')
clf.fit(word2vec_train, y_train)
word2vec_y_pred = clf.predict(word2vec_test)
print('Word2vec LR test accuracy %s' % metrics.accuracy_score(y_test, word2vec_y_pred))
#逻辑回归模型在测试集上的F1_Score
print('Word2vec LR test F1_score %s' % metrics.f1_score(y_test, word2vec_y_pred,average="macro"))

# TODO11: 使用bert + 逻辑回归训练模型，需要用gridsearchCV做交叉验证，并选择最好的超参数
clf = LogisticRegression()

from sklearn.model_selection import GridSearchCV

search_grid = {
    'C': [0.01, 1, 10, 100],
    'class_weight': [None, 'balanced']
}

grid_search = GridSearchCV(estimator = clf, 
                           param_grid = search_grid, 
                           cv = 5, 
                           n_jobs=-1, 
                           scoring='accuracy')

grid_result = grid_search.fit(bert_train, y_train[:len(y_train)//K])
print(f'Best parameters: {grid_result.best_params_}')
clf.fit(bert_train, y_train[:len(y_train)//K])
bert_y_pred = lr.predict(bert_test)
print('Bert LR test accuracy %s' % metrics.accuracy_score(y_test[:len(y_test)//K], bert_y_pred))
#逻辑回归模型在测试集上的F1_Score
print('Bert LR test F1_score %s' % metrics.f1_score(y_test[:len(y_test)//K], bert_y_pred,average="macro"))