semsim/FrequencyCounterFile.java at master · shainer/semsim · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
/*
 * Frequency counts from an external text file.
 *
 * Copyright (C) 2013 Lisa Vitolo <lisavitolo90@gmail.com>
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the Creative Commons
 * Attribution-NonCommercial-ShareAlike 3.0 license.
 * You should have received a copy of the license with this product.
 * Otherwise, visit http://creativecommons.org/licenses/by-nc-sa/3.0/
 *
 */
importjava.util.Map;
importjava.util.HashMap;
importjava.util.List;
importjava.math.BigInteger;

/*
 * The Takelab authors have grouped together the frequency counts for all the words found in the
 * training files in one text file. Here we read from that file in order to speed up things.
 *
 * The frequency file doesn't use any POS tag.
 */
publicclassFrequencyCounterFileimplementsFrequencyCounter
{
privateMap<String, BigInteger> frequencyCounts;
privateBigIntegertotalCount;

publicFrequencyCounterFile(Stringfile)
 {
frequencyCounts = newHashMap<>();
List<String> entries = IOUtils.readlines(file);

for (Stringentry : entries) {
String[] fields = entry.split(" ");

if (fields.length > 1) {
frequencyCounts.put(fields[0], newBigInteger(fields[1]));
 } else {
totalCount = newBigInteger(fields[0]); /* the first row is the total count */
 }
 }
 }

public@OverrideBigIntegergetFrequencyCount(Stringtoken, Stringtag)
 {
BigIntegerc = frequencyCounts.get(token);

/* This should not happen unless you use customized training files. */
if (c == null) {
returnnewBigInteger("0");
 }

returnc;
 }

public@OverrideBigIntegergetTotalCount()
 {
returntotalCount;
 }
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
/*
 * Frequency counts from an external text file.
 *
 * Copyright (C) 2013 Lisa Vitolo <lisavitolo90@gmail.com>
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the Creative Commons
 * Attribution-NonCommercial-ShareAlike 3.0 license.
 * You should have received a copy of the license with this product.
 * Otherwise, visit http://creativecommons.org/licenses/by-nc-sa/3.0/
 *
 */
importjava.util.Map;
importjava.util.HashMap;
importjava.util.List;
importjava.math.BigInteger;

/*
 * The Takelab authors have grouped together the frequency counts for all the words found in the
 * training files in one text file. Here we read from that file in order to speed up things.
 *
 * The frequency file doesn't use any POS tag.
 */
publicclassFrequencyCounterFileimplementsFrequencyCounter
{
privateMap<String, BigInteger> frequencyCounts;
privateBigIntegertotalCount;

publicFrequencyCounterFile(Stringfile)
 {
frequencyCounts = newHashMap<>();
List<String> entries = IOUtils.readlines(file);

for (Stringentry : entries) {
String[] fields = entry.split(" ");

if (fields.length > 1) {
frequencyCounts.put(fields[0], newBigInteger(fields[1]));
 } else {
totalCount = newBigInteger(fields[0]); /* the first row is the total count */
 }
 }
 }

public@OverrideBigIntegergetFrequencyCount(Stringtoken, Stringtag)
 {
BigIntegerc = frequencyCounts.get(token);

/* This should not happen unless you use customized training files. */
if (c == null) {
returnnewBigInteger("0");
 }

returnc;
 }

public@OverrideBigIntegergetTotalCount()
 {
returntotalCount;
 }
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
/*
 * Frequency counts from an external text file.
 *
 * Copyright (C) 2013 Lisa Vitolo <lisavitolo90@gmail.com>
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the Creative Commons
 * Attribution-NonCommercial-ShareAlike 3.0 license.
 * You should have received a copy of the license with this product.
 * Otherwise, visit http://creativecommons.org/licenses/by-nc-sa/3.0/
 *
 */
importjava.util.Map;
importjava.util.HashMap;
importjava.util.List;
importjava.math.BigInteger;

/*
 * The Takelab authors have grouped together the frequency counts for all the words found in the
 * training files in one text file. Here we read from that file in order to speed up things.
 *
 * The frequency file doesn't use any POS tag.
 */
publicclassFrequencyCounterFileimplementsFrequencyCounter
{
privateMap<String, BigInteger> frequencyCounts;
privateBigIntegertotalCount;

publicFrequencyCounterFile(Stringfile)
 {
frequencyCounts = newHashMap<>();
List<String> entries = IOUtils.readlines(file);

for (Stringentry : entries) {
String[] fields = entry.split(" ");

if (fields.length > 1) {
frequencyCounts.put(fields[0], newBigInteger(fields[1]));
 } else {
totalCount = newBigInteger(fields[0]); /* the first row is the total count */
 }
 }
 }

public@OverrideBigIntegergetFrequencyCount(Stringtoken, Stringtag)
 {
BigIntegerc = frequencyCounts.get(token);

/* This should not happen unless you use customized training files. */
if (c == null) {
returnnewBigInteger("0");
 }

returnc;
 }

public@OverrideBigIntegergetTotalCount()
 {
returntotalCount;
 }
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
/*
 * Frequency counts from an external text file.
 *
 * Copyright (C) 2013 Lisa Vitolo <lisavitolo90@gmail.com>
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the Creative Commons
 * Attribution-NonCommercial-ShareAlike 3.0 license.
 * You should have received a copy of the license with this product.
 * Otherwise, visit http://creativecommons.org/licenses/by-nc-sa/3.0/
 *
 */
importjava.util.Map;
importjava.util.HashMap;
importjava.util.List;
importjava.math.BigInteger;

/*
 * The Takelab authors have grouped together the frequency counts for all the words found in the
 * training files in one text file. Here we read from that file in order to speed up things.
 *
 * The frequency file doesn't use any POS tag.
 */
publicclassFrequencyCounterFileimplementsFrequencyCounter
{
privateMap<String, BigInteger> frequencyCounts;
privateBigIntegertotalCount;

publicFrequencyCounterFile(Stringfile)
 {
frequencyCounts = newHashMap<>();
List<String> entries = IOUtils.readlines(file);

for (Stringentry : entries) {
String[] fields = entry.split(" ");

if (fields.length > 1) {
frequencyCounts.put(fields[0], newBigInteger(fields[1]));
 } else {
totalCount = newBigInteger(fields[0]); /* the first row is the total count */
 }
 }
 }

public@OverrideBigIntegergetFrequencyCount(Stringtoken, Stringtag)
 {
BigIntegerc = frequencyCounts.get(token);

/* This should not happen unless you use customized training files. */
if (c == null) {
returnnewBigInteger("0");
 }

returnc;
 }

public@OverrideBigIntegergetTotalCount()
 {
returntotalCount;
 }
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
/*
 * Frequency counts from an external text file.
 *
 * Copyright (C) 2013 Lisa Vitolo <lisavitolo90@gmail.com>
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the Creative Commons
 * Attribution-NonCommercial-ShareAlike 3.0 license.
 * You should have received a copy of the license with this product.
 * Otherwise, visit http://creativecommons.org/licenses/by-nc-sa/3.0/
 *
 */
importjava.util.Map;
importjava.util.HashMap;
importjava.util.List;
importjava.math.BigInteger;

/*
 * The Takelab authors have grouped together the frequency counts for all the words found in the
 * training files in one text file. Here we read from that file in order to speed up things.
 *
 * The frequency file doesn't use any POS tag.
 */
publicclassFrequencyCounterFileimplementsFrequencyCounter
{
privateMap<String, BigInteger> frequencyCounts;
privateBigIntegertotalCount;

publicFrequencyCounterFile(Stringfile)
 {
frequencyCounts = newHashMap<>();
List<String> entries = IOUtils.readlines(file);

for (Stringentry : entries) {
String[] fields = entry.split(" ");

if (fields.length > 1) {
frequencyCounts.put(fields[0], newBigInteger(fields[1]));
 } else {
totalCount = newBigInteger(fields[0]); /* the first row is the total count */
 }
 }
 }

public@OverrideBigIntegergetFrequencyCount(Stringtoken, Stringtag)
 {
BigIntegerc = frequencyCounts.get(token);

/* This should not happen unless you use customized training files. */
if (c == null) {
returnnewBigInteger("0");
 }

returnc;
 }

public@OverrideBigIntegergetTotalCount()
 {
returntotalCount;
 }
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
/*
 * Frequency counts from an external text file.
 *
 * Copyright (C) 2013 Lisa Vitolo <lisavitolo90@gmail.com>
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the Creative Commons
 * Attribution-NonCommercial-ShareAlike 3.0 license.
 * You should have received a copy of the license with this product.
 * Otherwise, visit http://creativecommons.org/licenses/by-nc-sa/3.0/
 *
 */
importjava.util.Map;
importjava.util.HashMap;
importjava.util.List;
importjava.math.BigInteger;

/*
 * The Takelab authors have grouped together the frequency counts for all the words found in the
 * training files in one text file. Here we read from that file in order to speed up things.
 *
 * The frequency file doesn't use any POS tag.
 */
publicclassFrequencyCounterFileimplementsFrequencyCounter
{
privateMap<String, BigInteger> frequencyCounts;
privateBigIntegertotalCount;

publicFrequencyCounterFile(Stringfile)
 {
frequencyCounts = newHashMap<>();
List<String> entries = IOUtils.readlines(file);

for (Stringentry : entries) {
String[] fields = entry.split(" ");

if (fields.length > 1) {
frequencyCounts.put(fields[0], newBigInteger(fields[1]));
 } else {
totalCount = newBigInteger(fields[0]); /* the first row is the total count */
 }
 }
 }

public@OverrideBigIntegergetFrequencyCount(Stringtoken, Stringtag)
 {
BigIntegerc = frequencyCounts.get(token);

/* This should not happen unless you use customized training files. */
if (c == null) {
returnnewBigInteger("0");
 }

returnc;
 }

public@OverrideBigIntegergetTotalCount()
 {
returntotalCount;
 }
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
/*
 * Frequency counts from an external text file.
 *
 * Copyright (C) 2013 Lisa Vitolo <lisavitolo90@gmail.com>
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the Creative Commons
 * Attribution-NonCommercial-ShareAlike 3.0 license.
 * You should have received a copy of the license with this product.
 * Otherwise, visit http://creativecommons.org/licenses/by-nc-sa/3.0/
 *
 */
importjava.util.Map;
importjava.util.HashMap;
importjava.util.List;
importjava.math.BigInteger;

/*
 * The Takelab authors have grouped together the frequency counts for all the words found in the
 * training files in one text file. Here we read from that file in order to speed up things.
 *
 * The frequency file doesn't use any POS tag.
 */
publicclassFrequencyCounterFileimplementsFrequencyCounter
{
privateMap<String, BigInteger> frequencyCounts;
privateBigIntegertotalCount;

publicFrequencyCounterFile(Stringfile)
 {
frequencyCounts = newHashMap<>();
List<String> entries = IOUtils.readlines(file);

for (Stringentry : entries) {
String[] fields = entry.split(" ");

if (fields.length > 1) {
frequencyCounts.put(fields[0], newBigInteger(fields[1]));
 } else {
totalCount = newBigInteger(fields[0]); /* the first row is the total count */
 }
 }
 }

public@OverrideBigIntegergetFrequencyCount(Stringtoken, Stringtag)
 {
BigIntegerc = frequencyCounts.get(token);

/* This should not happen unless you use customized training files. */
if (c == null) {
returnnewBigInteger("0");
 }

returnc;
 }

public@OverrideBigIntegergetTotalCount()
 {
returntotalCount;
 }
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
/*
 * Frequency counts from an external text file.
 *
 * Copyright (C) 2013 Lisa Vitolo <lisavitolo90@gmail.com>
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the Creative Commons
 * Attribution-NonCommercial-ShareAlike 3.0 license.
 * You should have received a copy of the license with this product.
 * Otherwise, visit http://creativecommons.org/licenses/by-nc-sa/3.0/
 *
 */
importjava.util.Map;
importjava.util.HashMap;
importjava.util.List;
importjava.math.BigInteger;

/*
 * The Takelab authors have grouped together the frequency counts for all the words found in the
 * training files in one text file. Here we read from that file in order to speed up things.
 *
 * The frequency file doesn't use any POS tag.
 */
publicclassFrequencyCounterFileimplementsFrequencyCounter
{
privateMap<String, BigInteger> frequencyCounts;
privateBigIntegertotalCount;

publicFrequencyCounterFile(Stringfile)
 {
frequencyCounts = newHashMap<>();
List<String> entries = IOUtils.readlines(file);

for (Stringentry : entries) {
String[] fields = entry.split(" ");

if (fields.length > 1) {
frequencyCounts.put(fields[0], newBigInteger(fields[1]));
 } else {
totalCount = newBigInteger(fields[0]); /* the first row is the total count */
 }
 }
 }

public@OverrideBigIntegergetFrequencyCount(Stringtoken, Stringtag)
 {
BigIntegerc = frequencyCounts.get(token);

/* This should not happen unless you use customized training files. */
if (c == null) {
returnnewBigInteger("0");
 }

returnc;
 }

public@OverrideBigIntegergetTotalCount()
 {
returntotalCount;
 }
}