import json

import pandas as pd

df_all = pd.read_json('review.json', encoding='utf-8', lines=True)

print(df_all.shape, df_all.columns)

print(df_all.head())

# 用于提取信息的类

import nltk
from nltk.corpus import stopwords
stopwords = stopwords.words('english')
grammar = r"""
 NBAR:
    {<NN.*|JJ>*<NN.*>}  # Nouns and Adjectives, terminated with Nouns
 NP:
    {<NBAR>}
    {<NBAR><IN><NBAR>}  # Above, connected with in/of/etc...
"""
lemmatizer = nltk.WordNetLemmatizer()
stemmer = nltk.stem.porter.PorterStemmer()

class NounPhraseExtractor(object):

    def __init__(self, sentence):
        self.sentence = sentence

    def execute(self):
        # Taken from Su Nam Kim Paper...
        chunker = nltk.RegexpParser(grammar)
        #toks = nltk.regexp_tokenize(text, sentence_re)
        # #postoks = nltk.tag.pos_tag(toks)
        toks = nltk.word_tokenize(self.sentence)
        postoks = nltk.tag.pos_tag(toks)
        tree = chunker.parse(postoks)
        return tree

    def leaves(self, tree):
        """Finds NP (nounphrase) leaf nodes of a chunk tree."""
        for subtree in tree.subtrees(filter=lambda t: t.label() == 'NP'):
            yield subtree.leaves()

    def normalise(self, word):
        """Normalises words to lowercase and stems and lemmatizes it."""
        word = word.lower()
        word = stemmer.stem(word)
        word = lemmatizer.lemmatize(word)
        return word

    def acceptable_word(self, word):
        """Checks conditions for acceptable word: length, stopword."""
        accepted = bool(2 <= len(word) <= 40
                    and word.lower() not in stopwords)
        return accepted

    def get_terms(self,tree):
        for leaf in self.leaves(tree):
            term = [self.normalise(w) for w, t in leaf if self.acceptable_word(w)]
        yield term

    def extract(self):
        terms = self.get_terms(self.execute())
        matches = []
        for term in terms:
            for word in term:
                matches.append(word)
        return matches

# example
document = 'A novel device was designed to measure drainage dynamics of thin liquid films confined between a solid particle, an immiscible liquid droplet, and/or gas bubble. Equipped with a bimorph force sensor'
extract = NounPhraseExtractor(document)
extract.extract()

group_bus_id = df_all.groupby(['business_id'])

import time
from collections import defaultdict
import json

f = open('result_dict.txt', 'a+')


g_ct = 0

for business_id, df in group_bus_id:  # 遍历每个组
    start = time.time()
    business_info = defaultdict(dict)
    
    g_ct += 1
    if g_ct <= 192606:  # 控制读取个数
        continue
    
    print(g_ct, length)
    length = len(df)
    if length < 100:
        continue
    
    scores = []
    aspect_ct = defaultdict(int)
    aspect_reverse_index = defaultdict(list)  # 组内每个aspect对于(text, business_id的倒排表)
    
    for _, row in df.iterrows():
#         print('_'*32)
        
        
        try:
            starts = row['stars']
            review_id = row['review_id']
            text = row['text']
            
#             print(starts, review_id, text)
            scores.append(starts)
            extract_aspects = NounPhraseExtractor(text).extract()  # 对于每个评论生成aspect
            for extract_aspect in extract_aspects:
                aspect_reverse_index[extract_aspect].append((review_id, text))
                aspect_ct[extract_aspect] += 1
        except Exception as e:
            print(e)
            pass
    
    sorted_aspect_ct = sorted(aspect_ct.items(), key = lambda x: x[1], reverse=True)[:5]
    business_info[business_id]['aspects'] = [k for k, _ in sorted_aspect_ct]
    for k, _ in sorted_aspect_ct:
        business_info[business_id][k] = aspect_reverse_index[k]
    business_info[business_id]['scores'] = scores
            
#     print(business_info)
    f.write('{}\n'.format(json.dumps(business_info)))
    print(g_ct, length, time.time() - start)
    
f.close()

df_train = df_all[['text', 'stars']]

df_train = pd.read_csv('dataset.csv', sep='\t', nrows=3000)

text_list = df_train['text'].tolist()

# todo: 划分训练, 测试集

from sklearn.feature_extraction.text import TfidfVectorizer
tfidf_vectorizer = TfidfVectorizer()
tfidf_vectorizer.fit_transform(text_list)

train_X = tfidf_vectorizer.transform(text_list).toarray()
train_y = df_train['stars'].tolist()

from sklearn import datasets
iris = datasets.load_iris()

from sklearn.naive_bayes import GaussianNB
clf = GaussianNB()
clf = clf.fit(train_X, train_y)
y_pred=clf.predict(train_X)

# todo: 何判断一个评语有没有包含指定的 aspect

f = open('result_dict.txt', 'r')

for line in f:
    info = json.loads(line)
    # todo: 解析
    

f.close()