# 分数（15）
class my_CYK(object):
    def __init__(self, non_ternimal, terminal, rules_prob, start_prob):
        self.non_terminal = non_ternimal
        self.terminal = terminal
        self.rules_prob = rules_prob
        self.start_symbol = start_prob


    def parse_sentence(self, sentence):
        # TODO：parse the sentence with CYK algoritm
        
        sents = sentence.split()
        best_path = [[{} for _ in range(len(sents))] for _ in range(len(sents))]

        # initialization
        for i in range(len(sents)):
            for x in self.non_terminal:
                best_path[i][i][x] = {}
                if (sents[i],) in self.rules_prob[x].keys():
                    best_path[i][i][x]['prob'] = self.rules_prob[x][(sents[i],)]
                    best_path[i][i][x]['path'] = {'split':None, 'rule': sents[i]}
                else:
                    best_path[i][i][x]['prob'] = 0
                    best_path[i][i][x]['path'] = {'split':None, 'rule': None}

        # CKY recursive
        for l in range(1, len(sents)):
            for i in range(len(sents)-l):
                j = i + l
                for x in self.non_terminal:
                    tmp_best_x = {'prob':0, 'path':None}
                    for key, value in self.rules_prob[x].items():
                        if key[0] not in self.non_terminal: 
                            break
                        for s in range(i, j):
                            tmp_prob = value * best_path[i][s][key[0]]['prob'] * best_path[s+1][j][key[1]]['prob']
                            if tmp_prob > tmp_best_x['prob']:
                                tmp_best_x['prob'] = tmp_prob
                                tmp_best_x['path'] = {'split': s, 'rule': key}
                    best_path[i][j][x] = tmp_best_x
        self.best_path = best_path

        # TODO：print the result with tree structure
        self._parse_result(0, len(sents)-1, self.start_symbol)
        
        # TODO：print the probability for most probable parsing
        print("prob = ", self.best_path[0][len(sents)-1][self.start_symbol]['prob'])


    def _parse_result(self, left_idx, right_idx, root, ind=0):
        """
        print the result with tree- structure
        """
        node = self.best_path[left_idx][right_idx][root]
        if node['path']['split'] is not None:
            print('\t'*ind, (root, self.rules_prob[root].get(node['path']['rule'])))
            self._parse_result(left_idx, node['path']['split'], node['path']['rule'][0], ind+1)
            self._parse_result(node['path']['split']+1, right_idx, node['path']['rule'][1], ind+1)
        else:
            print('\t'*ind, (root, self.rules_prob[root].get((node['path']['rule'],))) )
            print('--->', node['path']['rule'])



def main(sentence):
    non_terminal = {'S', 'NP', 'VP', 'PP', 'DT', 'Vi', 'Vt', 'NN', 'IN'}
    start_symbol = 'S'
    terminal = {'sleeps', 'saw', 'boy', 'girl', 'dog', 'telescope', 'the', 'with', 'in'}
    rules_prob = {'S': {('NP', 'VP'): 1.0},
                  'VP': {('Vt', 'NP'): 0.8, ('VP', 'PP'): 0.2},
                  'NP': {('DT', 'NN'): 0.8, ('NP', 'PP'): 0.2},
                  'PP': {('IN', 'NP'): 1.0},
                  'Vi': {('sleeps',): 1.0},
                  'Vt': {('saw',): 1.0},
                  'NN': {('boy',): 0.1, ('girl',): 0.1,('telescope',): 0.3,('dog',): 0.5},
                  'DT': {('the',): 1.0},
                  'IN': {('with',): 0.6, ('in',): 0.4},
                }
    cyk = my_CYK(non_terminal, terminal, rules_prob, start_symbol)
    cyk.parse_sentence(sentence)

# TODO: 对该测试用例进行测试
if __name__ == "__main__":
    sentence = "the boy saw the dog with the telescope"
    main(sentence)

# TODO: 上面所写的算法的时间复杂度和空间复杂度分别是多少？
时间复杂度=O(), 空间复杂度=O()

import jieba
import jieba.posseg as pseg
import re
import datetime


# TODO： 从输入的“公司名”中提取主体
def main_extract(input_str,stop_word,d_4_delete,d_city_province):
    input_str = replace(input_str,d_city_province)
    #开始分词
    seg = pseg.cut(input_str)
    seg_lst = []
    for w in seg:
        elmt = w.word
        if elmt not in d_4_delete:
            seg_lst.append(elmt)
    seg_lst = remove_word(seg_lst,stop_word)
    seg_lst = city_prov_ahead(seg_lst,d_city_province)
    return seg_lst

    
#TODO：实现公司名称中地名提前
def city_prov_ahead(seg_lst,d_city_province):
    city_prov_lst = []
    for seg in seg_lst:
        if seg in d_city_province:
            city_prov_lst.append(seg)
            seg_lst.remove(seg)
    city_prov_lst.sort()
    return city_prov_lst+seg_lst


#TODO：替换特殊符号
def replace(com,d_city_province):
    #————————公司
    #地名
    com = re.sub(r'\(',"（",com)
    com = re.sub(r'\)',"）",com)
    pat = re.search(r'\（.+?\）',com)
    while pat:
        v = pat.group()[3:-3]
        start = pat.span()[0]
        end = pat.span()[1]
        if v not in d_city_province:
            com = com[:start]+com[end:]
        else:
            com = com[:start]+com[start+3:end-3]+com[end:]
        pat = re.search(r'\（.+?\）',com)
    #()（）
    com = re.sub(r'(\()*(\))*(\（)*(\）)*','',com)
    #全数字
    com = re.sub(r'^(\d)+$',"",com)
    return com


#TODO：初始化，加载词典
#输出“用来删除的字典”和“stop word”
def my_initial():
    fr1 = open(r"../data/dict/co_City_Dim.txt", encoding='utf-8')
    fr2 = open(r"../data/dict/co_Province_Dim.txt", encoding='utf-8')
    fr3 = open(r"../data/dict/company_business_scope.txt", encoding='utf-8')
    fr4 = open(r"../data/dict/company_suffix.txt", encoding='utf-8')
    #城市名
    lines1 = fr1.readlines()
    d_4_delete = []
    d_city_province = [re.sub(r'(\r|\n)*','',line) for line in lines1]
    #省份名
    lines2 = fr2.readlines()
    l2_tmp = [re.sub(r'(\r|\n)*','',line) for line in lines2]
    d_city_province.extend(l2_tmp)
    #公司后缀
    lines3 = fr3.readlines()
    l3_tmp = [re.sub(r'(\r|\n)*','',line) for line in lines3]
    lines4 = fr4.readlines()
    l4_tmp = [re.sub(r'(\r|\n)*','',line) for line in lines4]
    d_4_delete.extend(l4_tmp)
    #get stop_word
    fr = open(r'../data/dict/stopwords.txt', encoding='utf-8')   
    stop_word = fr.readlines()
    stop_word_after = [re.sub(r'(\r|\n)*','',stop_word[i]) for i in range(len(stop_word))]
    stop_word_after[-1] = stop_word[-1]
    stop_word = stop_word_after
    return d_4_delete,stop_word,d_city_province

# TODO：测试实体统一用例
d_4_delete,stop_word,d_city_province = my_initial()
company_name = "河北银行股份有限公司"
lst = main_extract(company_name,stop_word,d_4_delete,d_city_province)
company_name = ''.join(lst)  # 对公司名提取主体部分，将包含相同主体部分的公司统一为一个实体
print(company_name)

train_data = pd.read_csv('../data/info_extract/train_data.csv', encoding = 'gb2312', header=0)
train_data.head()

test_data = pd.read_csv('../data/info_extract/test_data.csv', encoding = 'gb2312', header=0)
test_data.head()

# 处理test数据，利用开源工具进行实体识别和并使用实体统一函数存储实体

import fool
import pandas as pd
from copy import copy


test_data = pd.read_csv('../data/info_extract/test_data.csv', encoding = 'gb2312', header=0)
test_data['ner'] = None
ner_id = 1001
ner_dict_new = {}  # 存储所有实体
ner_dict_reverse_new = {}  # 存储所有实体

for i in range(len(test_data)):
    sentence = copy(test_data.iloc[i, 1])
    words, ners = fool.analysis(sentence)
    ners[0].sort(key=lambda x:x[0], reverse=True)
    for start, end, ner_type, ner_name in ners[0]:
        if ner_type=='company' or ner_type=='person':
            company_main_name = ''.join(main_extract(ner_name,stop_word,d_4_delete,d_city_province))  # 提取公司主体名称
            if company_main_name not in ner_dict_new:
                ner_dict_new[company_main_name] = ner_id
                ner_id+=1
            sentence = sentence[:start] + ' ner_' + str(ner_dict_new[company_main_name]) + '_ ' + sentence[end-1:]
    test_data.iloc[i, -1] = sentence

X_test = test_data[['ner']]

# 处理train数据，利用开源工具进行实体识别和并使用实体统一函数存储实体
train_data = pd.read_csv('../data/info_extract/train_data.csv', encoding = 'gb2312', header=0)
train_data['ner'] = None

for i in range(len(train_data)):
    if train_data.iloc[i,:]['member1']=='0' and train_data.iloc[i,:]['member2']=='0':
        sentence = copy(train_data.iloc[i, 1])
        words, ners = fool.analysis(sentence)
        ners[0].sort(key=lambda x:x[0], reverse=True)
        for start, end, ner_type, ner_name in ners[0]:
            if ner_type=='company' or ner_type=='person':
                company_main_name = ''.join(main_extract(ner_name,stop_word,d_4_delete,d_city_province))  # 提取公司主体名称
                if company_main_name not in ner_dict_new:
                    ner_dict_new[company_main_name] = ner_id
                    ner_id+=1
                sentence = sentence[:start] + ' ner_' + str(ner_dict_new[company_main_name]) + '_ ' + sentence[end-1:]
        train_data.iloc[i, -1] = sentence
    else:
        train_data.iloc[i, -1] = re.sub(r'%s|%s'%(train_data.iloc[i,:]['member1'],train_data.iloc[i,:]['member2']), ' ner_1 ', train_data.iloc[i,:]['sentence'])

y = train_data.loc[:,['tag']]
train_num = len(train_data)
X_train = train_data[['ner']]

# 将train和test放在一起提取特征
X = pd.concat([X_train, X_test])

# code
from sklearn.feature_extraction.text import TfidfTransformer  
from sklearn.feature_extraction.text import CountVectorizer  
from pyltp import Segmentor

# 初始化实例
segmentor = Segmentor()  
# 加载模型，加载自定义词典
segmentor.load_with_lexicon('/Users/Badrain/Downloads/ltp_data_v3.4.0/cws.model', '../data/user_dict.txt')  

# 加载停用词
fr = open(r'../data/dict/stopwords.txt', encoding='utf-8')   
stop_word = fr.readlines()
stop_word = [re.sub(r'(\r|\n)*','',stop_word[i]) for i in range(len(stop_word))]


f = lambda x: ' '.join([for word in segmentor.segment(x) if word not in stop_word])
corpus=sample_data['sentence'].map(f).tolist()


from sklearn.feature_extraction.text import TfidfVectorizer
tfidf = TfidfVectorizer()
re = tfidf.fit_transform(corpus)

tfidf_feature = pd.DataFrame(tfidf_feature.toarray())
# print(re)

# -*- coding: utf-8 -*-
from pyltp import Parser
from pyltp import Segmentor
from pyltp import Postagger
import networkx as nx
import pylab
import re

postagger = Postagger() # 初始化实例
postagger.load_with_lexicon('/Users/Badrain/Downloads/ltp_data_v3.4.0/pos.model', '../data/user_dict.txt')  # 加载模型
segmentor = Segmentor()  # 初始化实例
segmentor.load_with_lexicon('/Users/Badrain/Downloads/ltp_data_v3.4.0/cws.model', '../data/user_dict.txt')  # 加载模型


# 实体符号加入分词词典
with open('../data/user_dict.txt', 'w') as fw:
    for v in ['一', '二', '三', '四', '五', '六', '七', '八', '九', '十']:
        fw.write( v + '号企业 ni\n')


def parse(s):
    """
    对语句进行句法分析，并返回句法结果
    parse_result：依存句法解析结果
    source：企业实体的词序号
    target：另一个企业实体的词序号
    keyword_pos：关键词词序号列表
    source_dep：企业实体依存句法类型
    target_dep：另一个企业实体依存句法类型
    """
    tmp_ner_dict = {}
    num_lst = ['一', '二', '三', '四', '五', '六', '七', '八', '九', '十']

    # 将公司代码替换为特殊称谓，保证分词词性正确
    for i, ner in enumerate(list(set(re.findall(r'(ner\_\d\d\d\d\_)', s)))):
        try:
            tmp_ner_dict[num_lst[i]+'号企业'] = ner
        except IndexError:
            return None, None, None, None, None, None
        s = s.replace(ner, num_lst[i]+'号企业')
    words = segmentor.segment(s)
    tags = postagger.postag(words)
    parser = Parser() # 初始化实例
    parser.load('/Users/Badrain/Downloads/ltp_data_v3.4.0/parser.model')  # 加载模型
    arcs = parser.parse(words, tags)  # 句法分析
    arcs_lst = list(map(list, zip(*[[arc.head, arc.relation] for arc in arcs])))
    
    # 句法分析结果输出
    parse_result = pd.DataFrame([[a,b,c,d] for a,b,c,d in zip(list(words),list(tags), arcs_lst[0], arcs_lst[1])], index = range(1,len(words)+1))
    parser.release()  # 释放模型
    
    # 能找到两个企业以上才返回结果，目前简化，只考虑前两家企业的关系
    try:
        source = list(words).index('一号企业')+1
        target = list(words).index('二号企业')+1
        source_dep = arcs_lst[1][source-1]
        target_dep = arcs_lst[1][target-1]
    except:
        return None, None, None, None, None, None
    
    # 投资关系关键词
    key_words = ["收购","竞拍","转让","扩张","并购","注资","整合","并入","竞购","竞买","支付","收购价","收购价格","承购","购得","购进",
             "购入","买进","买入","赎买","购销","议购","函购","函售","抛售","售卖","销售","转售"]
    keyword_pos = [list(words).index(w)+1 if w in list(words) else -1 for w in key_words]
    
    return parse_result, source, target, keyword_pos, source_dep, target_dep


def shortest_path(arcs_ret, source, target):
    """
    求出两个词最短依存句法路径，不存在路径返回-1
    """
    G=nx.DiGraph()
    # 为这个网络添加节点...
    for i in list(arcs_ret.index):
        G.add_node(i)
    # 在网络中添加带权中的边...
    for i in list(arcs_ret.index):
        G.add_edges_from([( arcs_ret.iloc[i-1, 2], i )])
        G.add_edges_from([( i, arcs_ret.iloc[i-1, 2] )])

    try:
        distance=nx.shortest_path_length(G,source,target)
        #     print('源节点为0，终点为7,最短距离：', distance)
        return distance
    except:
        return -1


def get_parse_feature(s):
    """
    综合上述函数汇总句法分析特征
    """
    parse_result, source, target, keyword_pos, source_dep, target_dep = parse(s)
    if parse_result is None:
        return [-1]*59
    features = []
    features.append(shortest_path(parse_result, source, target))
    keyword_feature = []
    for p in keyword_pos:
        if p==-1:
            keyword_feature.append(-1)
            keyword_feature.append(-1)
        else:
            keyword_feature.append(shortest_path(parse_result, source, p))
            keyword_feature.append(shortest_path(parse_result, target, p))
    features.extend(keyword_feature)
    features.extend([source_dep, target_dep])
    return features

# 对所有样本统一派生特征

f = lambda x: get_parse_feature(x)
parse_feature = X.map(f)

whole_feature = []
for i in range(len(parse_feature)):
    whole_feature.append(list(tfidf_feature.iloc[i,:]) + parse_feature[i])

whole_feature = pd.DataFrame(whole_feature)

# 建立分类器进行分类
from sklearn.ensemble import RandomForestClassifier
from sklearn import preprocessing
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import GridSearchCV

# TODO：定义需要遍历的参数
parameters = { 'C':np.logspace(-3,3,7)}

# TODO：选择模型
lr = LogisticRegression()

# TODO：利用GridSearchCV搜索最佳参数
clf = GridSearchCV(lr, parameters, cv=5)
clf.fit(train_x, y)

# TODO：对Test_data进行分类
predict = clf.predict(test_x)
predict_prob = clf.predict_proba(test_x)


# TODO：保存结果
# 答案提交在submit目录中，命名为info_extract_submit.csv和info_extract_entity.csv。
# info_extract_entity.csv格式为：第一列是实体编号，第二列是实体名（实体统一的多个实体名用“|”分隔）
# info_extract_submit.csv格式为：第一列是关系中实体1的编号，第二列为关系中实体2的编号。

 

from py2neo import Node, Relationship, Graph

graph = Graph(
    "http://localhost:7474", 
    username="neo4j", 
    password="person"
)

for v in relation_list:
    a = Node('Company', name=v[0])
    b = Node('Company', name=v[1])
    
    # 本次不区分投资方和被投资方，无向图
    r = Relationship(a, 'INVEST', b)
    s = a | b | r
    graph.create(s)
    r = Relationship(b, 'INVEST', a)
    s = a | b | r
    graph.create(s)

# TODO：查询某节点的3层投资关系

import jieba
import pandas as pd

# 找出test_data.csv中前25条样本所有的人物名称，以及人物所在文档的上下文内容
test_data = pd.read_csv('../data/info_extract/test_data.csv', encoding = 'gb2312', header=0)

# 存储人物以及上下文信息（key为人物ID，value为人物名称、人物上下文内容）
person_name = {}

# 观察上下文的窗口大小
window = 10  

# 遍历前25条样本
for i in range(25):
    sentence = copy(test_data.iloc[i, 1])
    words, ners = fool.analysis(sentence)
    ners[0].sort(key=lambda x:x[0], reverse=True)
    for start, end, ner_type, ner_name in ners[0]:
        if ner_type=='person':
            # TODO：提取上下文
            person_name[ner_name] = person_name.get(ner_name)+sentence[max(0, idx-window):min(len(sentence)-1, idx+window)]



# 利用爬虫得到每个人物名称对应的URL
# TODO：找到每个人物实体的词条内容。

# TODO：将样本中人物上下文与爬取词条结果进行对比，选择最接近的词条。



# 输出结果
pd.DataFrame(result_data).to_csv('../submit/entity_disambiguation_submit.csv', index=False)