Text_processor/text_processing.py at main · heeahcodes/Text_processor · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
defmain():
importspacy
fromcollectionsimportCounter#to count the frequency of a named entity

nlp=spacy.load("en_core_web_sm") #loads the english language module for spacy
#loads the novel with utf-8 encoding to process both regular and special characters safely
withopen("Bleak House.txt", "r",
encoding="utf-8") asfile:
#spilts texts by double newlines to separate content into smaller chunks
text_chunks=file.read().split("\n\n")
#cleans whitespace, removes empty strings
text_chunks= [chunk.strip() forchunkintext_chunksifchunk.strip()]

""" The pipe method was used to chunk the text [ Bleak House by Charles Dickens] as the character count
 exceeded spaCy's character limit of 1000000 characters
 """
location_list= [] #list to append all the recognized entities in the text
#n_process unlocks all cores for faster processing, surpasses GIL
fordocinnlp.pipe(text_chunks, batch_size=30, n_process=-1): #parsing size limited to chunks of 30
forentindoc.ents:
ifent.label_in ["GPE", "LOC"]: #entity labels for geopolitical entities and non-GPE locations
location_list.append(ent.text)
location_counts=Counter(location_list)

#displays output in the form of a list with names and the frequency of appearance
print(f"found locations --- ")
forlocation, countinlocation_counts.most_common(30):
print(f"{location}:{count} mentions")

if__name__=='__main__': # boilerplate
main()
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
defmain():
importspacy
fromcollectionsimportCounter#to count the frequency of a named entity

nlp=spacy.load("en_core_web_sm") #loads the english language module for spacy
#loads the novel with utf-8 encoding to process both regular and special characters safely
withopen("Bleak House.txt", "r",
encoding="utf-8") asfile:
#spilts texts by double newlines to separate content into smaller chunks
text_chunks=file.read().split("\n\n")
#cleans whitespace, removes empty strings
text_chunks= [chunk.strip() forchunkintext_chunksifchunk.strip()]

""" The pipe method was used to chunk the text [ Bleak House by Charles Dickens] as the character count
 exceeded spaCy's character limit of 1000000 characters
 """
location_list= [] #list to append all the recognized entities in the text
#n_process unlocks all cores for faster processing, surpasses GIL
fordocinnlp.pipe(text_chunks, batch_size=30, n_process=-1): #parsing size limited to chunks of 30
forentindoc.ents:
ifent.label_in ["GPE", "LOC"]: #entity labels for geopolitical entities and non-GPE locations
location_list.append(ent.text)
location_counts=Counter(location_list)

#displays output in the form of a list with names and the frequency of appearance
print(f"found locations --- ")
forlocation, countinlocation_counts.most_common(30):
print(f"{location}:{count} mentions")

if__name__=='__main__': # boilerplate
main()
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
defmain():
importspacy
fromcollectionsimportCounter#to count the frequency of a named entity

nlp=spacy.load("en_core_web_sm") #loads the english language module for spacy
#loads the novel with utf-8 encoding to process both regular and special characters safely
withopen("Bleak House.txt", "r",
encoding="utf-8") asfile:
#spilts texts by double newlines to separate content into smaller chunks
text_chunks=file.read().split("\n\n")
#cleans whitespace, removes empty strings
text_chunks= [chunk.strip() forchunkintext_chunksifchunk.strip()]

""" The pipe method was used to chunk the text [ Bleak House by Charles Dickens] as the character count
 exceeded spaCy's character limit of 1000000 characters
 """
location_list= [] #list to append all the recognized entities in the text
#n_process unlocks all cores for faster processing, surpasses GIL
fordocinnlp.pipe(text_chunks, batch_size=30, n_process=-1): #parsing size limited to chunks of 30
forentindoc.ents:
ifent.label_in ["GPE", "LOC"]: #entity labels for geopolitical entities and non-GPE locations
location_list.append(ent.text)
location_counts=Counter(location_list)

#displays output in the form of a list with names and the frequency of appearance
print(f"found locations --- ")
forlocation, countinlocation_counts.most_common(30):
print(f"{location}:{count} mentions")

if__name__=='__main__': # boilerplate
main()
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
defmain():
importspacy
fromcollectionsimportCounter#to count the frequency of a named entity

nlp=spacy.load("en_core_web_sm") #loads the english language module for spacy
#loads the novel with utf-8 encoding to process both regular and special characters safely
withopen("Bleak House.txt", "r",
encoding="utf-8") asfile:
#spilts texts by double newlines to separate content into smaller chunks
text_chunks=file.read().split("\n\n")
#cleans whitespace, removes empty strings
text_chunks= [chunk.strip() forchunkintext_chunksifchunk.strip()]

""" The pipe method was used to chunk the text [ Bleak House by Charles Dickens] as the character count
 exceeded spaCy's character limit of 1000000 characters
 """
location_list= [] #list to append all the recognized entities in the text
#n_process unlocks all cores for faster processing, surpasses GIL
fordocinnlp.pipe(text_chunks, batch_size=30, n_process=-1): #parsing size limited to chunks of 30
forentindoc.ents:
ifent.label_in ["GPE", "LOC"]: #entity labels for geopolitical entities and non-GPE locations
location_list.append(ent.text)
location_counts=Counter(location_list)

#displays output in the form of a list with names and the frequency of appearance
print(f"found locations --- ")
forlocation, countinlocation_counts.most_common(30):
print(f"{location}:{count} mentions")

if__name__=='__main__': # boilerplate
main()
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
defmain():
importspacy
fromcollectionsimportCounter#to count the frequency of a named entity

nlp=spacy.load("en_core_web_sm") #loads the english language module for spacy
#loads the novel with utf-8 encoding to process both regular and special characters safely
withopen("Bleak House.txt", "r",
encoding="utf-8") asfile:
#spilts texts by double newlines to separate content into smaller chunks
text_chunks=file.read().split("\n\n")
#cleans whitespace, removes empty strings
text_chunks= [chunk.strip() forchunkintext_chunksifchunk.strip()]

""" The pipe method was used to chunk the text [ Bleak House by Charles Dickens] as the character count
 exceeded spaCy's character limit of 1000000 characters
 """
location_list= [] #list to append all the recognized entities in the text
#n_process unlocks all cores for faster processing, surpasses GIL
fordocinnlp.pipe(text_chunks, batch_size=30, n_process=-1): #parsing size limited to chunks of 30
forentindoc.ents:
ifent.label_in ["GPE", "LOC"]: #entity labels for geopolitical entities and non-GPE locations
location_list.append(ent.text)
location_counts=Counter(location_list)

#displays output in the form of a list with names and the frequency of appearance
print(f"found locations --- ")
forlocation, countinlocation_counts.most_common(30):
print(f"{location}:{count} mentions")

if__name__=='__main__': # boilerplate
main()
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
defmain():
importspacy
fromcollectionsimportCounter#to count the frequency of a named entity

nlp=spacy.load("en_core_web_sm") #loads the english language module for spacy
#loads the novel with utf-8 encoding to process both regular and special characters safely
withopen("Bleak House.txt", "r",
encoding="utf-8") asfile:
#spilts texts by double newlines to separate content into smaller chunks
text_chunks=file.read().split("\n\n")
#cleans whitespace, removes empty strings
text_chunks= [chunk.strip() forchunkintext_chunksifchunk.strip()]

""" The pipe method was used to chunk the text [ Bleak House by Charles Dickens] as the character count
 exceeded spaCy's character limit of 1000000 characters
 """
location_list= [] #list to append all the recognized entities in the text
#n_process unlocks all cores for faster processing, surpasses GIL
fordocinnlp.pipe(text_chunks, batch_size=30, n_process=-1): #parsing size limited to chunks of 30
forentindoc.ents:
ifent.label_in ["GPE", "LOC"]: #entity labels for geopolitical entities and non-GPE locations
location_list.append(ent.text)
location_counts=Counter(location_list)

#displays output in the form of a list with names and the frequency of appearance
print(f"found locations --- ")
forlocation, countinlocation_counts.most_common(30):
print(f"{location}:{count} mentions")

if__name__=='__main__': # boilerplate
main()
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
defmain():
importspacy
fromcollectionsimportCounter#to count the frequency of a named entity

nlp=spacy.load("en_core_web_sm") #loads the english language module for spacy
#loads the novel with utf-8 encoding to process both regular and special characters safely
withopen("Bleak House.txt", "r",
encoding="utf-8") asfile:
#spilts texts by double newlines to separate content into smaller chunks
text_chunks=file.read().split("\n\n")
#cleans whitespace, removes empty strings
text_chunks= [chunk.strip() forchunkintext_chunksifchunk.strip()]

""" The pipe method was used to chunk the text [ Bleak House by Charles Dickens] as the character count
 exceeded spaCy's character limit of 1000000 characters
 """
location_list= [] #list to append all the recognized entities in the text
#n_process unlocks all cores for faster processing, surpasses GIL
fordocinnlp.pipe(text_chunks, batch_size=30, n_process=-1): #parsing size limited to chunks of 30
forentindoc.ents:
ifent.label_in ["GPE", "LOC"]: #entity labels for geopolitical entities and non-GPE locations
location_list.append(ent.text)
location_counts=Counter(location_list)

#displays output in the form of a list with names and the frequency of appearance
print(f"found locations --- ")
forlocation, countinlocation_counts.most_common(30):
print(f"{location}:{count} mentions")

if__name__=='__main__': # boilerplate
main()
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
defmain():
importspacy
fromcollectionsimportCounter#to count the frequency of a named entity

nlp=spacy.load("en_core_web_sm") #loads the english language module for spacy
#loads the novel with utf-8 encoding to process both regular and special characters safely
withopen("Bleak House.txt", "r",
encoding="utf-8") asfile:
#spilts texts by double newlines to separate content into smaller chunks
text_chunks=file.read().split("\n\n")
#cleans whitespace, removes empty strings
text_chunks= [chunk.strip() forchunkintext_chunksifchunk.strip()]

""" The pipe method was used to chunk the text [ Bleak House by Charles Dickens] as the character count
 exceeded spaCy's character limit of 1000000 characters
 """
location_list= [] #list to append all the recognized entities in the text
#n_process unlocks all cores for faster processing, surpasses GIL
fordocinnlp.pipe(text_chunks, batch_size=30, n_process=-1): #parsing size limited to chunks of 30
forentindoc.ents:
ifent.label_in ["GPE", "LOC"]: #entity labels for geopolitical entities and non-GPE locations
location_list.append(ent.text)
location_counts=Counter(location_list)

#displays output in the form of a list with names and the frequency of appearance
print(f"found locations --- ")
forlocation, countinlocation_counts.most_common(30):
print(f"{location}:{count} mentions")

if__name__=='__main__': # boilerplate
main()