summarization_with_strategy_TFIDF.py

import spacy
import math
from app.summary.summary_strategy_interface import ISummaryStrategy


class SummaryTFIDF(ISummaryStrategy):
    nlpGer = spacy.load('de_core_news_sm')

    def __init__(self):
        self._id = "tfidf"
        super().__init__()

    @property
    def id(self):
        return self._id

    def frequency_matrix(self, summary):
        """
        This method creates a tf-idf-matrix which is a list with all sentences
            containing a list with all words in the sentence and their
            frequency as value

        :param summary: given text to summarize
        :returns: freq_matrix: frequency matrix
        """
        freq_matrix = {}

        stop_words = self.nlpGer.Defaults.stop_words

        for sent in summary:
            # dictionary with 'words' as the key
            # and their 'frequency' as the value
            freq_table = {}

            words = [word.lemma_.lower() for word in sent if
                     word.text.isalnum()]  # Lemmatize the word

            for word in words:
                if word not in stop_words:  # Reject stopWords
                    if word in freq_table:
                        freq_table[word] += 1
                    else:
                        freq_table[word] = 1

            freq_matrix[sent] = freq_table

        return freq_matrix

    @staticmethod
    def tf_matrix(freq_matrix):
        """
        This method calculates the term frequency for every word

        :param freq_matrix: frequency matrix
        :returns: tf_matrix: a list with all sentences containing a list with
            all words and their term frequency
        """
        tf_matrix = {}

        for sent, freq_table in freq_matrix.items():
            # dictionary with 'word' itself as the key and its TF as the value
            tf_table = {}

            total_words_in_sentence = len(freq_table)
            for word, count in freq_table.items():
                tf_table[word] = count / total_words_in_sentence

            tf_matrix[sent] = tf_table

        return tf_matrix

    @staticmethod
    def sentences_per_words(freq_matrix):
        """
        This methods returns a list with all words and how often a word is
            mentioned in a sentence

        :param freq_matrix: frequency matrix
        :returns: sent_per_words: sentences per words list
        """
        sent_per_words = {}

        for sent, f_table in freq_matrix.items():
            for word, count in f_table.items():
                if word in sent_per_words:
                    sent_per_words[word] += 1
                else:
                    sent_per_words[word] = 1

        return sent_per_words

    @staticmethod
    def idf_matrix(freq_matrix, sent_per_words, total_sentences):
        """
        This methods calculates a idf score for every word

        :param freq_matrix: frequency matrix
        :param sent_per_words: sentences per words list
        :param total_sentences: total sentences list
        :returns: idf_matrix: list of all sentences containing a list of
            all words in this sentence with their idf value
        """
        idf_matrix = {}

        for sent, f_table in freq_matrix.items():
            idf_table = {}

            for word in f_table.keys():
                idf_table[word] = math.log10(
                    total_sentences / float(sent_per_words[word]))

            idf_matrix[sent] = idf_table

        return idf_matrix

    @staticmethod
    def tf_idf_matrix(tf_matrix, idf_matrix):
        """
        This methods calculates a tf-idf-score for every word

        :param tf_matrix: tf-matrix
        :param idf_matrix: idf-matrix
        :returns: tf_idf_matrix: list of all sentences containing a list of
            all words with their tf-idf value
        """
        tf_idf_matrix = {}

        for (sent1, f_table1), (sent2, f_table2) in zip(
                tf_matrix.items(), idf_matrix.items()):

            tf_idf_table = {}

            for (word1, tf_value), (word2, idf_value) in zip(f_table1.items(),
                                                             f_table2.items()):
                tf_idf_table[word1] = float(tf_value * idf_value)

            tf_idf_matrix[sent1] = tf_idf_table

        return tf_idf_matrix

    @staticmethod
    def score_sentences(tf_idf_matrix):
        """
        This methods calculates a sentence score for every sentence based on
            the tf-idf-matrix

        :param tf_idf_matrix: tf-idf-matrix
        :returns: sentence_score: list of all sentences with sentence score
        """
        sentence_score = {}

        for sent, f_table in tf_idf_matrix.items():
            total_tfidf_score_per_sentence = 0

            total_words_in_sentence = len(f_table)
            for word, tf_idf_score in f_table.items():
                total_tfidf_score_per_sentence += tf_idf_score

            if total_words_in_sentence != 0:
                sentence_score[sent] = total_tfidf_score_per_sentence / \
                                       total_words_in_sentence

        return sentence_score

    @staticmethod
    def average_score(sentence_score):
        """
        This method calculates the average sentence score

        :param sentence_score: list of sentences with sentence score
        :returns: average_sent_score: average sentence score
        """

        total_score = 0
        for sent in sentence_score:
            total_score += sentence_score[sent]

        average_sent_score = (total_score / len(sentence_score))

        return average_sent_score

    @staticmethod
    def create_summary(sentences, sentence_score, threshold):
        """
        This method returns a summary with all sentences having a higher
            sentence score than the threshold

        :param sentences: list of all sentences
        :param sentence_score: list of sentences with sentence score
        :param threshold: threshold for sentence score
        :returns: summary: generated summary
        """
        summary = ''

        for sentence in sentences:
            if sentence in sentence_score and sentence_score[sentence] >= (
                    threshold):
                summary += " " + sentence.text

        return summary[1:]

    def summarize(self, text: str, max_length: int) -> str:
        text = self.nlpGer(text)

        # put all sentences in a list
        sentences = list(text.sents)
        total_sentences = len(sentences)

        # generate frequency matrix
        freq_matrix = self.frequency_matrix(sentences)

        # generate term frequency matrix
        tf_matrix = self.tf_matrix(freq_matrix)

        # identify number of sentences with a specific word
        num_sent_per_words = self.sentences_per_words(freq_matrix)

        # generate id-frequency matrix
        idf_matrix = self.idf_matrix(
            freq_matrix, num_sent_per_words, total_sentences)

        # generate tf-idf-matrix
        tf_idf_matrix = self.tf_idf_matrix(tf_matrix, idf_matrix)

        # generate sentence score for every sentence
        sentence_scores = self.score_sentences(tf_idf_matrix)

        # set threshold to average score
        threshold = self.average_score(sentence_scores)

        # summary
        summary = self.create_summary(
            sentences, sentence_scores, threshold)

        return summary