ML.NaiveBayes

  1from .model import Model
  2
  3from nltk import pos_tag
  4from nltk import NaiveBayesClassifier
  5from nltk import classify
  6from nltk import TweetTokenizer
  7
  8from nltk.sentiment.vader import SentimentIntensityAnalyzer
  9from nltk.stem import WordNetLemmatizer
 10from nltk.corpus import stopwords
 11
 12from random import shuffle
 13from statistics import mean
 14
 15import joblib
 16
 17class NaiveBayes(Model):
 18    def __init__(self) -> None:
 19        """"
 20        Builds Vectoriser and stopwords object to be used in  _preprocess.
 21        """
 22        super().__init__()
 23        # TD: should try to make stopwords static, and data needed for objects static also.
 24        self._sia = SentimentIntensityAnalyzer()
 25        self._lemmatizer = WordNetLemmatizer()
 26
 27        self._tokenizer = TweetTokenizer(preserve_case=False,
 28                                        strip_handles=True,
 29                                        reduce_len=True)
 30        
 31        self._TAGMAP = {'V' : 'v', 'J' : 'a', 'N' : 'n', 'R' : 'r' }
 32
 33        self._STOPWORDS = set(stopwords.words("english"))
 34            
 35    def _preprocess(self, tweet):
 36        """
 37        Cleans data before sentiment analysis, including removing stopwords and alphas.
 38        Args:
 39            tweet(str): tweet data of string
 40        Returns:
 41            data (list(str)): data that is vectorised and cleaned
 42        """
 43        data = self._tokenizer.tokenize(tweet)
 44
 45        data = [token for token in data if token.isalpha() and token not in self._STOPWORDS]
 46
 47        # (low) TD: Pull request pos tag (tagset) to work with lemmatize
 48        data = pos_tag(data)
 49    
 50        # (low) TD: refact if data + lemmatize + pos_tag
 51        
 52        data = [self._lemmatizer.lemmatize(token, self._TAGMAP.get(pos, 'n')) for token, pos in data]
 53        
 54        return data
 55    
 56    def _features(self, tweet):
 57        """
 58        Calcs the VADER score (from lexicon data)
 59        Args:
 60            tweet(str): tweet data of string
 61        Returns:
 62            features(dict): dictionary of pos and comp score as keys, to 0-1 float.
 63        """
 64        data = self._preprocess(tweet)
 65        
 66        if not data:
 67            return {}
 68
 69        features = {}
 70        positive_scores = []
 71        compound_scores = []
 72
 73        for word in data:
 74            positive_scores.append(self._sia.polarity_scores(word)["pos"])
 75            compound_scores.append(self._sia.polarity_scores(word)["compound"])
 76        features['pos_score'] = mean(positive_scores)
 77        features['comp_score'] = mean(compound_scores)
 78        return features
 79    
 80    def _trainmodel(self):
 81        """
 82        Using NLP's NaiveBayes probablity classifier, use labeled data to build 
 83        a pkl file.
 84        """
 85        features = []
 86        for tweet in self.pos_data:
 87            features.append((self._features(tweet), 'p'))
 88
 89        for tweet in self.neg_data:
 90            features.append((self._features(tweet), 'n'))
 91
 92        shuffle(features)
 93        classifier = NaiveBayesClassifier.train(labeled_featuresets=features[:1500])
 94        joblib.dump(classifier, r'backend\ML\models\NaiveBayes.pkl')
 95
 96    async def predict(self, tweet):
 97        """
 98        By using the trained pkl file in models, classify a single tweet
 99        Args:
100            tweet (str) : text data inside of tweet.
101        Returns:
102            'p' to indicate positive and 'n' to indicate negative
103        """
104        print(tweet)
105        tweet = self._features(tweet)
106        result = joblib.load(r'backend\ML\models\NaiveBayes.pkl').classify(tweet)
107        return result
108    
109if __name__ == "__main__": 
110    from nltk import download
111    download(['wordnet'])