ACID/tokenize_data.py at main · deeplearningplus/ACID · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
fromdatasetsimportload_dataset, load_from_disk
fromtransformersimportAutoTokenizer
importtorch.utils.data
importos

# load tokenizer
tokenizer=AutoTokenizer.from_pretrained('opt-seq-pubmed-tokenizer')
max_length=1024

deftokenize_function(examples):
returntokenizer(examples["text"], max_length=max_length, truncation=True)

cache_dir='./tmp'# Cache directory

data_files= {'train': ["data/trn.csv"],
'test' : ["data/val.csv"]}
extension='csv'
raw_datasets=load_dataset(extension, data_files=data_files, cache_dir=cache_dir)

preprocessing_num_workers=4
overwrite_cache=False
removed_columns= ['text']

tokenized_datasets=raw_datasets.map(
tokenize_function,
#batched=True,
num_proc=preprocessing_num_workers,
remove_columns=removed_columns,
load_from_cache_file=notoverwrite_cache,
desc="Running tokenizer on dataset",
)

tokenized_datasets.save_to_disk('tokenized_data')

# Remove cache_dir
#os.system("rm -rf ./tmp")

##ds = load_from_disk('TMP')
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
fromdatasetsimportload_dataset, load_from_disk
fromtransformersimportAutoTokenizer
importtorch.utils.data
importos

# load tokenizer
tokenizer=AutoTokenizer.from_pretrained('opt-seq-pubmed-tokenizer')
max_length=1024

deftokenize_function(examples):
returntokenizer(examples["text"], max_length=max_length, truncation=True)

cache_dir='./tmp'# Cache directory

data_files= {'train': ["data/trn.csv"],
'test' : ["data/val.csv"]}
extension='csv'
raw_datasets=load_dataset(extension, data_files=data_files, cache_dir=cache_dir)

preprocessing_num_workers=4
overwrite_cache=False
removed_columns= ['text']

tokenized_datasets=raw_datasets.map(
tokenize_function,
#batched=True,
num_proc=preprocessing_num_workers,
remove_columns=removed_columns,
load_from_cache_file=notoverwrite_cache,
desc="Running tokenizer on dataset",
)

tokenized_datasets.save_to_disk('tokenized_data')

# Remove cache_dir
#os.system("rm -rf ./tmp")

##ds = load_from_disk('TMP')
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
fromdatasetsimportload_dataset, load_from_disk
fromtransformersimportAutoTokenizer
importtorch.utils.data
importos

# load tokenizer
tokenizer=AutoTokenizer.from_pretrained('opt-seq-pubmed-tokenizer')
max_length=1024

deftokenize_function(examples):
returntokenizer(examples["text"], max_length=max_length, truncation=True)

cache_dir='./tmp'# Cache directory

data_files= {'train': ["data/trn.csv"],
'test' : ["data/val.csv"]}
extension='csv'
raw_datasets=load_dataset(extension, data_files=data_files, cache_dir=cache_dir)

preprocessing_num_workers=4
overwrite_cache=False
removed_columns= ['text']

tokenized_datasets=raw_datasets.map(
tokenize_function,
#batched=True,
num_proc=preprocessing_num_workers,
remove_columns=removed_columns,
load_from_cache_file=notoverwrite_cache,
desc="Running tokenizer on dataset",
)

tokenized_datasets.save_to_disk('tokenized_data')

# Remove cache_dir
#os.system("rm -rf ./tmp")

##ds = load_from_disk('TMP')
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
fromdatasetsimportload_dataset, load_from_disk
fromtransformersimportAutoTokenizer
importtorch.utils.data
importos

# load tokenizer
tokenizer=AutoTokenizer.from_pretrained('opt-seq-pubmed-tokenizer')
max_length=1024

deftokenize_function(examples):
returntokenizer(examples["text"], max_length=max_length, truncation=True)

cache_dir='./tmp'# Cache directory

data_files= {'train': ["data/trn.csv"],
'test' : ["data/val.csv"]}
extension='csv'
raw_datasets=load_dataset(extension, data_files=data_files, cache_dir=cache_dir)

preprocessing_num_workers=4
overwrite_cache=False
removed_columns= ['text']

tokenized_datasets=raw_datasets.map(
tokenize_function,
#batched=True,
num_proc=preprocessing_num_workers,
remove_columns=removed_columns,
load_from_cache_file=notoverwrite_cache,
desc="Running tokenizer on dataset",
)

tokenized_datasets.save_to_disk('tokenized_data')

# Remove cache_dir
#os.system("rm -rf ./tmp")

##ds = load_from_disk('TMP')
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
fromdatasetsimportload_dataset, load_from_disk
fromtransformersimportAutoTokenizer
importtorch.utils.data
importos

# load tokenizer
tokenizer=AutoTokenizer.from_pretrained('opt-seq-pubmed-tokenizer')
max_length=1024

deftokenize_function(examples):
returntokenizer(examples["text"], max_length=max_length, truncation=True)

cache_dir='./tmp'# Cache directory

data_files= {'train': ["data/trn.csv"],
'test' : ["data/val.csv"]}
extension='csv'
raw_datasets=load_dataset(extension, data_files=data_files, cache_dir=cache_dir)

preprocessing_num_workers=4
overwrite_cache=False
removed_columns= ['text']

tokenized_datasets=raw_datasets.map(
tokenize_function,
#batched=True,
num_proc=preprocessing_num_workers,
remove_columns=removed_columns,
load_from_cache_file=notoverwrite_cache,
desc="Running tokenizer on dataset",
)

tokenized_datasets.save_to_disk('tokenized_data')

# Remove cache_dir
#os.system("rm -rf ./tmp")

##ds = load_from_disk('TMP')
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
fromdatasetsimportload_dataset, load_from_disk
fromtransformersimportAutoTokenizer
importtorch.utils.data
importos

# load tokenizer
tokenizer=AutoTokenizer.from_pretrained('opt-seq-pubmed-tokenizer')
max_length=1024

deftokenize_function(examples):
returntokenizer(examples["text"], max_length=max_length, truncation=True)

cache_dir='./tmp'# Cache directory

data_files= {'train': ["data/trn.csv"],
'test' : ["data/val.csv"]}
extension='csv'
raw_datasets=load_dataset(extension, data_files=data_files, cache_dir=cache_dir)

preprocessing_num_workers=4
overwrite_cache=False
removed_columns= ['text']

tokenized_datasets=raw_datasets.map(
tokenize_function,
#batched=True,
num_proc=preprocessing_num_workers,
remove_columns=removed_columns,
load_from_cache_file=notoverwrite_cache,
desc="Running tokenizer on dataset",
)

tokenized_datasets.save_to_disk('tokenized_data')

# Remove cache_dir
#os.system("rm -rf ./tmp")

##ds = load_from_disk('TMP')
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
fromdatasetsimportload_dataset, load_from_disk
fromtransformersimportAutoTokenizer
importtorch.utils.data
importos

# load tokenizer
tokenizer=AutoTokenizer.from_pretrained('opt-seq-pubmed-tokenizer')
max_length=1024

deftokenize_function(examples):
returntokenizer(examples["text"], max_length=max_length, truncation=True)

cache_dir='./tmp'# Cache directory

data_files= {'train': ["data/trn.csv"],
'test' : ["data/val.csv"]}
extension='csv'
raw_datasets=load_dataset(extension, data_files=data_files, cache_dir=cache_dir)

preprocessing_num_workers=4
overwrite_cache=False
removed_columns= ['text']

tokenized_datasets=raw_datasets.map(
tokenize_function,
#batched=True,
num_proc=preprocessing_num_workers,
remove_columns=removed_columns,
load_from_cache_file=notoverwrite_cache,
desc="Running tokenizer on dataset",
)

tokenized_datasets.save_to_disk('tokenized_data')

# Remove cache_dir
#os.system("rm -rf ./tmp")

##ds = load_from_disk('TMP')
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
fromdatasetsimportload_dataset, load_from_disk
fromtransformersimportAutoTokenizer
importtorch.utils.data
importos

# load tokenizer
tokenizer=AutoTokenizer.from_pretrained('opt-seq-pubmed-tokenizer')
max_length=1024

deftokenize_function(examples):
returntokenizer(examples["text"], max_length=max_length, truncation=True)

cache_dir='./tmp'# Cache directory

data_files= {'train': ["data/trn.csv"],
'test' : ["data/val.csv"]}
extension='csv'
raw_datasets=load_dataset(extension, data_files=data_files, cache_dir=cache_dir)

preprocessing_num_workers=4
overwrite_cache=False
removed_columns= ['text']

tokenized_datasets=raw_datasets.map(
tokenize_function,
#batched=True,
num_proc=preprocessing_num_workers,
remove_columns=removed_columns,
load_from_cache_file=notoverwrite_cache,
desc="Running tokenizer on dataset",
)

tokenized_datasets.save_to_disk('tokenized_data')

# Remove cache_dir
#os.system("rm -rf ./tmp")

##ds = load_from_disk('TMP')