텍스트 마이닝 - Bag of Words

from collections import Counter
import nltk
from nltk import word_tokenize
nltk.download('punkt')

# 텍스트
text = """Yesterday I went fishing. I don't fish that often, 
so I didn't catch any fish. I was told I'd enjoy myself, 
but it didn't really seem that fun."""

# 토큰화
tokens = word_tokenize(text)

# 모든 단어를 소문자화
lower_tokens = [t.lower() for t in tokens]

# Counter화
bow_simple = Counter(lower_tokens)

# 상위 10개의 단어 추출
print(bow_simple.most_common(10))

from sklearn.feature_extraction.text import CountVectorizer

text = [
    'This is the first document.',
    'This is the second second document.',
    'And the third one.',
]
vect = CountVectorizer()
vect.fit(text)
vect.vocabulary_

X = vect.fit_transform(text)
print(vect.get_feature_names())
print(X.toarray())

from gensim.corpora.dictionary import Dictionary
from nltk.tokenize import word_tokenize
my_documents = ['When the screenwriter John Ridley pitched “The Other History of the DC Universe,” a five-part comic book series that looks at pivotal events through the perspectives of several nonwhite DC heroes, he knew Black Lightning would be at its center. Ridley was 11 when he met the hero in 1977.',
                'This was a Black man who ostensibly looked like me with his own series,” Mr. Ridley said during a recent interview. An added bonus',
                'Mr. Ridley’s career includes writing for television and film, which earned him an Academy Award in 2014 for Best Adapted Screenplay for “12 Years a Slave.” But he is no stranger to comics. He wrote “The American Way,” which was published in 2006, about a group of heroes in the 1960s and their reaction to a Black member joining the team, and a sequel in 2017.'
                ]
punctuations= '?:!.,;“'
tokens = []
for sentence in my_documents:
  sentence_words = word_tokenize(sentence)
  for word in sentence_words:
    if word in punctuations:
      sentence_words.remove(word)
  tokens.append(sentence_words)

tokens[0][:10]

from gensim.corpora.dictionary import Dictionary

dictionary = Dictionary(tokens)
print(dictionary[19]) # key-value로 확인한다. 
print(len(dictionary)) # 학습된 단어의 수
Ridley_id = dictionary.token2id.get("Ridley")
print(dictionary.get(Ridley_id))
corpus = [dictionary.doc2bow(article) for article in tokens]
print(corpus[2][:5])

from collections import defaultdict
import itertools

doc = corpus[2]
bow_doc = sorted(doc, key=lambda w: w[1], reverse=True)
for word_id, word_count in bow_doc[:5]:
    print(dictionary.get(word_id), word_count)


total_word_count = defaultdict(int)
for word_id, word_count in itertools.chain.from_iterable(corpus):
    total_word_count[word_id] += word_count

print("\n")
# 단어수가 많은 순대로 재 정렬
sorted_word_count = sorted(total_word_count.items(), key=lambda w: w[1], reverse=True) 

# 정렬된 단어를 내림차순으로 출력한다. 
for word_id, word_count in sorted_word_count[:5]:
    print(dictionary.get(word_id), word_count)

from gensim.models.tfidfmodel import TfidfModel

tfidf = TfidfModel(corpus)
tfidf_weights = tfidf[doc]

# 빈번하게 자주 나오는 단어 출력
print(tfidf_weights[:5])

# Sort the weights from highest to lowest: sorted_tfidf_weights
sorted_tfidf_weights = sorted(tfidf_weights, key=lambda w: w[1], reverse=True)

# Print the top 5 weighted words
for term_id, weight in sorted_tfidf_weights[:5]:
    print(dictionary.get(term_id), weight)