text-processing/FirstModule.py at master · kblanks/text-processing · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
'''
Created on Oct 27, 2018

@author: Kyle
'''

# import libraries
fromsklearn.feature_extraction.textimportCountVectorizer,TfidfVectorizer
frompandasimportDataFrame
fromnumpyimportasarray
fromyellowbrick.textimportFreqDistVisualizer
fromyellowbrick.text.tsneimportTSNEVisualizer

# define functions
defread_text(text_name):
withopen(text_name,'r') asf: book=f.read()
f.close()
returnbook

defdoVisualizer(featNames,vector,numTerms=10):
visualizer=FreqDistVisualizer(features=featNames,n=numTerms)
visualizer.fit(vector)
visualizer.poof()

# set up data
text_1=read_text('LOTR1.txt')
text_2=read_text('LOTR2.txt')
text_3=read_text('LOTR3.txt')
text= [text_1,text_2,text_3]

lotrDF=DataFrame()

# basic text level features
lotrDF['text'] =text
lotrDF['char_count'] =lotrDF['text'].apply(len)
lotrDF['word_count']=lotrDF['text'].apply(lambdax: len(x.split()))
lotrDF['word_density']=lotrDF['char_count']/(lotrDF['word_count']+1)

# count vectorizer
count_vec=CountVectorizer(stop_words='english', analyzer='word')
count_fit=count_vec.fit(lotrDF['text'])
vector_count=count_fit.transform(lotrDF['text'])
count_feat=count_vec.get_feature_names()
count_set=set(count_feat)
count_freqs=zip(count_feat,vector_count.sum(axis=0).tolist()[0])

fellowship_count_vec=CountVectorizer(stop_words='english', analyzer='word')
fellowship_vector=fellowship_count_vec.fit_transform([text_1])
fellowship_feat=fellowship_count_vec.get_feature_names()
fellowship_set=set(fellowship_feat)

towers_count_vec=CountVectorizer(stop_words='english', analyzer='word')
towers_vector=towers_count_vec.fit_transform([text_2])
towers_feat=towers_count_vec.get_feature_names()
towers_set=set(towers_feat)

return_count_vec=CountVectorizer(stop_words='english', analyzer='word')
return_vector=return_count_vec.fit_transform([text_3])
return_feat=return_count_vec.get_feature_names()
return_set=set(return_feat)

#unique word set
fellowship_unique=fellowship_set.difference(towers_set.union(return_set))
towers_unique=towers_set.difference(fellowship_set.union(return_set))
return_unique=return_set.difference(fellowship_set.union(towers_set))

# tfidf vectorizer
tfidf_vec=TfidfVectorizer(analyzer='word',norm='l2',use_idf=True,smooth_idf=True,sublinear_tf=False)
tfidf_fit=tfidf_vec.fit(lotrDF['text'])
vector_tfidf=tfidf_fit.transform(lotrDF['text'])
tfidf_feat=tfidf_vec.get_feature_names()
tfidf_freqs=zip(tfidf_feat,tfidf_vec.idf_)

fellowship_unique_list= []

foriteminfellowship_unique:
fellowship_unique_list.append(fellowship_vector.sum(axis=0).tolist()[0][fellowship_count_vec.vocabulary_[item]])

# output
print(sorted(zip(fellowship_unique,fellowship_unique_list), key=lambdax: -x[1])[:5])
print(sorted(count_freqs, key=lambdax: -x[1])[:5])
print(lotrDF['char_count'])
print(lotrDF['word_count'])
print(lotrDF['word_density'])
#doVisualizer(count_feat, vector_count)