bakeoff-sample/run_textrank_ver4.py

import pandas as pd
import argparse
import os
import re
import csv
import spacy
import json
import numpy as np
import logging

import traceback

from summa.summarizer import summarize
from gensim.models import Word2Vec
from gensim.utils import simple_preprocess
from gensim.parsing.preprocessing import remove_stopwords

from rouge import Rouge
from nltk.corpus import stopwords

from sklearn.feature_extraction.text import TfidfVectorizer

stopwords = set(stopwords.words('english'))
nlp = spacy.load('en_core_web_sm')

cache = {}

class NpEncoder(json.JSONEncoder):
    def default(self, obj):
        if isinstance(obj, np.integer):
            return int(obj)
        elif isinstance(obj, np.floating):
            return float(obj)
        elif isinstance(obj, np.ndarray):
            return obj.tolist()
        else:
            return super(NpEncoder, self).default(obj)

def build_vectorizer(path):
    """Create the idf, bigram indices and train the word2vec model
    Args:
        path (string): the path to the directory that contains
            'questions.csv' and 'answers.csv'
    Returns:
        Dict: a dictionary whose keys are the unigrams and bigrams
            (>=2 occurrence); the values are the idf scores
        Word2Vec: the trained gensim Word2Vec model
    """
    # questions
    corpus = []

    for filename in os.listdir(path):
        filepath = os.path.join(path, filename)
        


        with open(filepath) as json_file:
            try:
                question = json.load(json_file)
            except Exception as ee:
                continue


            question_text = question['question_text']
            corpus.append(question_text)
            
            answers = question['answers']


            for idx in range(len(answers)):
                answer = answers[idx]
                answer_text = answer['answer_text']
                corpus.append(answer_text)

    # train w2v
    model = Word2Vec([simple_preprocess(text.lower()) for text in corpus],
                     size=300, window=10)

    # find bigrams, idf
    vectorizer = TfidfVectorizer(ngram_range=(1,2), min_df=2, stop_words='english')
    vectorizer.fit(corpus)
    vocab = vectorizer.vocabulary_
    idf = vectorizer.idf_

    idf_dict = dict([(word, idf[vocab[word]]) for word in vocab])
    return idf_dict, model

def find_similar_phrases(phrase_vectors_a, phrase_vectors_b, phrase_similarity_threshold=0.85):
    """Return similar phrases
    """
    similar_phrases = {}
    for word_vector_pair_a in phrase_vectors_a:
        word_a, pv_a = word_vector_pair_a
        for word_vector_pair_b in phrase_vectors_b:
            word_b, pv_b = word_vector_pair_b
            cs = cosine_sim(pv_a, pv_b)

            if cs > phrase_similarity_threshold:
                if word_b in similar_phrases:
                    cs_old = similar_phrases[word_b]
                    if cs > cs_old:
                        similar_phrases[word_b] = cs
                else:
                    similar_phrases[word_b] = cs

    return [(w,similar_phrases[w]) for w in similar_phrases]

def cosine_sim(veca, vecb):
    """Cosine similarity of two vectors
    """
    lena = np.linalg.norm(veca)
    lenb = np.linalg.norm(vecb)
    if lena < 1e-6 or lenb < 1e-6:
        return 0.0
    else:
        return np.dot(veca, vecb) / lena / lenb

def phrase2vec(phrase, idf, w2v):
    """Compute the vector representation of a phrase.
    Sum of word vectors weighted by their idf's
    Args:
        phrase (string): the input phrase (space separated)
        idf (Dict): the idf dictionary
        w2v (Word2Vec): the word2vec model
    Returns:
        ndarray: a 300d representation vector
    """
    global cache
    phrase = phrase.lower()
    if phrase in cache:
        return cache[phrase]
    result = np.zeros(300)
    for token in phrase.split(' '):
        if token in w2v.wv and token in idf:
            result += w2v.wv[token] * idf[token]

    return result

def text2vec(text, idf, w2v):
    """Compute the vector representation of a paragraph.
    Sum of the unigrams and bigrams' phrase vectors. Preprocess
    with gensim.utils.simple_preprocess (dropping puncts)
    Args:
        text (string): the input text
        idf (Dict): the idf dictionary
        w2v (Word2Vec): the word2vec model
    Returns:
        ndarray: a 300d representation vector
    """
    if text in cache:
        return cache[text]

    tokens = simple_preprocess(remove_stopwords(text.lower()))
    bigrams = []
    for idx in range(len(tokens) - 1):
        bigram = tokens[idx] + ' ' + tokens[idx + 1]
        if bigram in idf:
            bigrams.append(bigram)

    ngrams = tokens + bigrams
    phrase_vectors = [phrase2vec(phrase, idf, w2v) for phrase in ngrams]

    phrase_vector_pairs = [(ngrams[i], phrase_vectors[i]) for i in range(len(ngrams))]

    text_vector = sum([np.zeros(300)] + phrase_vectors)
    cache[text] = (text_vector, phrase_vector_pairs)

    return (text_vector, phrase_vector_pairs)

def get_coverage(summaries, answer_text, answer_ids, phrase_similarity_threshold=0.85, sentence_similarity_threshold=0.5):
    """ Compute the coverage of each summary line.
    Args:
        summaries (List of str): the summary lines
        answer_text (List of str): the answers' original text
        answer_ids (List of str): the answers' id
    Returns:
        List of List of str: the answerId that each summary covered
        List of List of str: the informative tokens of each line
    """
    results = []
    informative_tokens = []
    for summary in summaries:
        summary_results = []
        summary_informative_tokens = {}
        logging.debug('Summary: {}'.format(summary))
        for cand_answer_text, aid in zip(answer_text, answer_ids):
            summary_vector, summary_phrase_vectors = text2vec(summary, idf, w2v)
            cand_answer_vector, cand_answer_phrase_vectors = text2vec(cand_answer_text, idf, w2v)

            logging.debug('Answer ID: {}'.format(aid))
            logging.debug('Answer   : {}'.format(cand_answer_text))

            sentence_similarity = cosine_sim(summary_vector, cand_answer_vector)
            logging.debug('Sentence Similarity   : {}'.format(sentence_similarity))

            if sentence_similarity >= sentence_similarity_threshold:
                summary_results.append((aid, sentence_similarity))

                similar_phrases = find_similar_phrases(summary_phrase_vectors, cand_answer_phrase_vectors, phrase_similarity_threshold=phrase_similarity_threshold)
                summary_informative_tokens[aid] = [re.sub('\s+', '_', phrase_score_pair[0]) for phrase_score_pair in similar_phrases]

        results.append(summary_results)
        informative_tokens.append(summary_informative_tokens)



    # def get_tokens(text):
    #     unigrams = simple_preprocess(text.lower())
    #     bigrams = [unigrams[i] + '_' + unigrams[i + 1]
    #                for i in range(len(unigrams) - 1)]
    #     return [token for token in unigrams + bigrams if token not in stopwords]
    #
    # all_tokens = [get_tokens(line) for line in summaries]
    # informative_tokens = []
    # for sid, tokens in enumerate(all_tokens):
    #     other_tokens = set([])
    #     for oid, others in enumerate(all_tokens):
    #         if oid != sid:
    #             other_tokens |= set(others)
    #     informative_tokens.append(list(set(tokens) - other_tokens))
    #
    # results = [[] for _ in summaries]
    # for sid, tokens in enumerate(informative_tokens):
    #     for text, aid in zip(answer_text, answer_ids):
    #         answer_tokens = get_tokens(text)
    #         if len(set(tokens) & set(answer_tokens)) > 0:
    #             results[sid].append(aid)
    #
    # # clean the informative tokens
    # for sid, tokens in enumerate(informative_tokens):
    #     new_tokens = []
    #     for token in tokens:
    #         if len(set(token.split('_')) & stopwords) == 0:
    #             new_tokens.append(token)
    #     informative_tokens[sid] = new_tokens
    #
    # print(results)
    # print(informative_tokens)

    return results, informative_tokens

def clean_sentence(text):
    """ Clean an input sentence
    Args:
        text (str): the sentence
    Returns:
        text (str): the cleaned sentence
    """
    sent = nlp(text)
    # clean the first few characters
    result = text
    for i, token in enumerate(sent):
        if token.pos_ not in ['PUNCT', 'X', 'SYM']:
            result = sent[i:].text
            break

    # capitalize the first character
    if len(result) > 0:
        result = result[0].upper() + result[1:]

    return result

def drop_unwanted_sentences(answers):
    """ Remove unwanted sentences and concatenate into one string
    Args:
        answers (list of str): the answers to be cleaned
    Returns:
        str: the cleaned and concatenated string
    """
    all_sents = []
    for text in answers:
        sents = nlp(text, disable=['tagger', 'ner']).sents
        for sent in sents:
            if len(sent) <= 30:
                all_sents.append(sent.text)

    return '\n'.join(all_sents)

def do_textrank(question, num_sent=5, num_words=100, phrase_similarity_threshold=0.85, sentence_similarity_threshold=0.5):
    """ Perform TextRank to generate the summary items with coverage information

    Args:
        question (json): question and answers

        {
            "question_id": "1dsbhnd88nrea800",
            "question_text": "I have an interview on Thursday what questions will they ask, if anyone has applied there in San Diego",
            "question_code": "",
            "question_topics": ["INTERVIEW"],
            "company_id": 11180,
            "company": "Sam's Club",
            "num_answers": 1,
            "answers": [{
                "answer_id": "1e0rqjv09q45d800",
                "answer_date": "2020-02-12T04:30:51.913",
                "job_title": "Inventory Control Supervisor",
                "job_location": "Princeton, NJ",
                "answer_text": "Tell us of the most difficult interaction you had with a customer and how did you positively turn the customer around?",
                "votes": []
             }]
        }

        num_sent (int, optional): the maximal number of sentences in the summary
        num_words (int, optional): the maximal number of words in the summary

    Returns:
        Dict: the summary result containing the summary items,
            coverage info, and related answers.
    """

    qid = question['question_id']
    question_text = question['question_text']
    question_code = question['question_code']
    question_topics = question['question_topics']

    answers = question['answers']

    if len(answers) < num_sent:
        return None

    answer_dict = {}

    for idx in range(len(answers)):
        answer = answers[idx]

        answer_id = answer['answer_id']
        answer_text = answer['answer_text']

        answer_dict[answer_id] = answer_text

    answer_ids = list(answer_dict.keys())
    answer_texts = list(answer_dict.values())
    
    result = {'question_id': qid,
              'question_text': question_text,
              'question_code': question_code,
              'question_topics': question_topics,
              'num_answers': len(answers)}

    # print(result)
    # input('press any key to continue')
    logging.debug("Input:" + str(answer_texts))
    full_text = drop_unwanted_sentences(answer_texts)
    logging.debug("Text:" + full_text)
    ans = summarize(full_text, words=num_words * 2)
    summary = []
    summary_lines = []

    stats = {}
    stats['num_summaries'] = 0
    stats['num_answers'] = len(answers)
    stats['num_supporting_answers'] = 0
    stats['num_summaries_with_zero_supporting_answers'] = 0
    stats['num_answers_with_zero_matching_tokens'] = 0


    for idx, line in enumerate(ans.split('\n')[:num_sent]):
        summary_lines.append(clean_sentence(line))

    stats['num_summaries'] = len(summary_lines)
    coverage, tokens = get_coverage(summary_lines, answer_texts, answer_ids, phrase_similarity_threshold=phrase_similarity_threshold, sentence_similarity_threshold=sentence_similarity_threshold)

    for idx, line in enumerate(summary_lines):
        support = len(coverage[idx])

        stats['num_supporting_answers'] = stats['num_supporting_answers'] + support

        if support == 0:
            stats['num_summaries_with_zero_supporting_answers'] = stats['num_summaries_with_zero_supporting_answers'] + 1

        summary_item = {}
        summary_item['text'] = line
        summary_item['support'] = support
        summary_item['coverage'] = []
        for aid, similarity in coverage[idx]:
            answer_item = {}
            answer_text = answer_dict[aid]
            answer_item['answer_id'] = aid
            answer_item['similarity'] = similarity
            answer_item['answer_text'] = answer_text
            summary_item['coverage'].append(answer_item)


            # Identify matching tokens
            answer_matching_tokens = []

            for token in tokens[idx][aid]:

                tre = token.lower().replace('_','\s+')
                matching_token = {}
                matching_token['token'] = token
                spans = []
                matching_token['spans'] = spans


                for m in re.finditer(tre, answer_text.lower()):
                    span = {}
                    span['text'] = answer_text[m.start():m.end()]
                    span['start'] = m.start()
                    span['end'] = m.end()
                    spans.append(span)

                if len(spans) > 0:
                    answer_matching_tokens.append(matching_token)

            answer_item['matching_tokens'] = answer_matching_tokens
            if len(answer_matching_tokens) == 0:
                stats['num_answers_with_zero_matching_tokens'] = stats['num_answers_with_zero_matching_tokens'] + 1

        # summary_item['tokens'] = tokens[idx]

        summary.append(summary_item)


    result['summary'] = summary


    rouge = Rouge()
    scores = rouge.get_scores(ans, full_text)[0]

    stats['ROUGE-1'] = scores['rouge-1']['f']
    stats['ROUGE-2'] = scores['rouge-2']['f']
    stats['ROUGE-L'] = scores['rouge-l']['f']
    result['stats'] = stats

    logging.debug(result)
    return result


def find_questions(keywords, topics, qids):
    c = questions['content'].str.contains('')
    if len(keywords) > 0:
        ck = questions['content'].str.contains('')
        for keyword in keywords:
            ck = ck & questions['content'].str.contains(keyword)
        c = c & ck

    if len(topics) > 0:
        ct = questions['topics'].str.contains(topics[0]) | questions['questionCode'].str.contains(topics[0])
        for topic in topics[1:]:
            ct = ct | ( questions['topics'].str.contains(topic) | questions['questionCode'].str.contains(topic) )
        c = c & ct

    if len(qids) > 0:
        qt = questions['questionId'].str.contains('')
        for qid in qids:
            qt = qt & questions['questionId'].str.contains(qid)
        c = c & qt

    return questions[c]

def write_json_to_file(data, file_name):
    with open(file_name, 'w') as fo:
        json.dump(data, fo, cls=NpEncoder)


def summarize_with_textrank(path,
                            num_sent=5,
                            num_words=100,
                            phrase_similarity_threshold=0.725,
                            sentence_similarity_threshold=0.65):

    results = []

    stats = {}
    stats['question_count'] = 0
    stats['summarized_question_count'] = 0
    stats['total_num_summaries'] = 0
    stats['total_num_answers'] = 0
    stats['total_num_supporting_answers'] = 0
    stats['total_num_summaries_with_zero_supporting_answers'] = 0
    stats['total_num_answers_with_zero_matching_tokens'] = 0
    stats['avg_ROUGE-1'] = 0
    stats['avg_ROUGE-2'] = 0
    stats['avg_ROUGE-L'] = 0

    for filename in os.listdir(path):
        filepath = os.path.join(path, filename)
        
    
        with open(filepath) as json_file:
            try:
                question = json.load(json_file)
            except Exception as ee:
                continue

        qid = question['question_id']
        
        if stats['question_count'] % 100 == 0:
            logging.info("Processed {} questions summarized {}".format(stats['question_count'],stats['summarized_question_count']))

        stats['question_count'] = stats['question_count'] + 1
        try:
            result = do_textrank(question, num_sent, num_words, phrase_similarity_threshold=phrase_similarity_threshold, sentence_similarity_threshold=sentence_similarity_threshold)


            if result != None:
                stats['avg_ROUGE-1'] = ( stats['avg_ROUGE-1'] * stats['summarized_question_count'] + result['stats']['ROUGE-1'] ) / ( stats['summarized_question_count'] + 1)
                stats['avg_ROUGE-2'] = ( stats['avg_ROUGE-2'] * stats['summarized_question_count'] + result['stats']['ROUGE-2'] ) / ( stats['summarized_question_count'] + 1)
                stats['avg_ROUGE-L'] = ( stats['avg_ROUGE-L'] * stats['summarized_question_count'] + result['stats']['ROUGE-L'] ) / ( stats['summarized_question_count'] + 1)
                stats['summarized_question_count'] = stats['summarized_question_count'] + 1
                stats['total_num_summaries'] = stats['total_num_summaries'] + result['stats']['num_summaries']
                stats['total_num_answers'] =  stats['total_num_answers'] + result['stats']['num_answers']
                stats['total_num_supporting_answers'] = stats['total_num_supporting_answers'] + result['stats']['num_supporting_answers']
                stats['total_num_summaries_with_zero_supporting_answers'] = stats['total_num_summaries_with_zero_supporting_answers'] + result['stats']['num_summaries_with_zero_supporting_answers']
                stats['total_num_answers_with_zero_matching_tokens'] = stats['total_num_answers_with_zero_matching_tokens'] + result['stats']['num_answers_with_zero_matching_tokens']

                results.append(result)
                write_json_to_file(result, "output/" + qid + ".json")
            else:
                continue
                # write_json_to_file({"message": "no summary created"}, "../data/output/" + qid + ".json")


        except Exception as exception:
            traceback.print_exc()
            logging.error("Exception on question: " + qid)

    logging.info("Final statistics {}".format(stats))

if __name__ == '__main__':
    # python run_textrank.py --keywords interview,precefiss --num_sentences 5 --num_words 100
    parser = argparse.ArgumentParser()
    parser.add_argument("--num_sentences", type=int, default=5)
    parser.add_argument("--num_words", type=int, default=100)
    parser.add_argument("--phrase_similarity_threshold", type=float, default=0.85)
    parser.add_argument("--sentence_similarity_threshold", type=float, default=0.5)
    parser.add_argument("--log", type=str, default=None)
    hp = parser.parse_args()

    # Initialize logging
    logging_level = logging.INFO if hp.log is None else hp.log.upper()
    logging.basicConfig(format='%(levelname)s:%(message)s', level=logging_level)


    # build idf and w2v models if necessary
    if os.path.exists('models/idf.json') and os.path.exists('models/indeed_qa.model'):
        idf = json.load(open('models/idf.json'))
        w2v = Word2Vec.load('models/indeed_qa.model')
    else:
        idf, w2v = build_vectorizer('data')
        json.dump(idf, open('models/idf.json', 'w'))
        w2v.save('models/indeed_qa.model')

    # summarize
    summarize_with_textrank('data',
                            num_sent=hp.num_sentences,
                            num_words=hp.num_words,
                            phrase_similarity_threshold=hp.phrase_similarity_threshold,
                            sentence_similarity_threshold=hp.sentence_similarity_threshold)