Add hugging face dataset examples.

.gitignore

@@ -1,5 +1,6 @@
 project_env/
 .DS_Store
+dataset/*
 
 # from hydra
 outputs/ 

README.rst

@@ -61,7 +61,8 @@ choose the features that you like. This flexibility is one of the best
 * Code formatting.
 * Unit tests and continuous integration.
 * Packaging and distribution.
-* Remove development.
+* Remote development.
+* Creating and sharing datasets with Hugging Face.
 
 The accompanying 
 `slides <https://docs.google.com/presentation/d/1D1_JywMl2rjaeuVzpykPBOJsDIuwQKGOJB4EFZjej2s/edit#slide=id.g2eaa4b61f15_0_1346>`__ 

examples/configs/hf_dataset.yaml

@@ -0,0 +1,29 @@
+# python examples/create_huggingface_dataset.py hf_token=YOUR_TOKEN
+hydra:
+ job:
+ chdir: True # change to output folder
+ job_logging:
+ formatters:
+ simple:
+ format: '[%(levelname)s] - %(message)s'
+
+repo_id: bezzam/dummy-dataset
+seed: 0
+test_size: 0.15
+hf_token:
+
+data_dir:
+ audio:
+ dir: dataset/data_audio
+ type: wav
+ image:
+ dir: dataset/data_images
+ type: png
+ text:
+ dir: dataset/data_text
+ type: txt
+ label:
+ file: dataset/data_labels.csv
+ label: True
+
+stratify_by_column: label

examples/create_huggingface_dataset.py

@@ -0,0 +1,232 @@
+"""
+We will create a dataset with images, audios, and text data
+so that you can see how various data types can be pushed to
+Hugging Face!
+
+The default configuration is in `examples/configs/hf_dataset.yaml`:
+
+```bash
+# install
+pip install datasets huggingface_hub soundfile
+
+# make a WRITE token on HuggingFace: https://huggingface.co/settings/tokens
+
+# run
+python examples/create_huggingface_dataset.py \
+hf_token=... \
+```
+"""
+
+import hydra
+from hydra.utils import to_absolute_path
+import os
+import time
+import glob
+import numpy as np
+import soundfile as sf
+from PIL import Image as PILImage
+from datasets import Dataset, Image, Audio, ClassLabel
+from omegaconf import open_dict
+from huggingface_hub import upload_file
+import re
+import pandas as pd
+
+
+# -- helper functions
+def convert(text):
+ return int(text) if text.isdigit() else text.lower()
+
+
+def alphanum_key(key):
+ return [convert(c) for c in re.split("([0-9]+)", key)]
+
+
+def natural_sort(arr):
+ return sorted(arr, key=alphanum_key)
+
+
+@hydra.main(version_base=None, config_path="configs", config_name="hf_dataset")
+def main(config):
+
+ start_time = time.time()
+
+ # extract and check parameters
+ repo_id = config.repo_id
+ hf_token = config.hf_token
+ test_size = config.test_size
+
+ assert repo_id is not None, "Please provide a Hugging Face repo_id."
+ assert hf_token is not None, "Please provide a Hugging Face token."
+
+ # to absolute path, as needed by Hugging Face upload
+ for data in config.data_dir:
+ if "dir" in config.data_dir[data]:
+ config.data_dir[data]["dir"] = to_absolute_path(config.data_dir[data]["dir"])
+ elif "file" in config.data_dir[data]:
+ config.data_dir[data]["file"] = to_absolute_path(config.data_dir[data]["file"])
+
+ # Step 1: Check data (create dummy data if not present)
+ n_files = 100 # number of dummy files to create
+ for data in config.data_dir:
+
+ # for directory of data
+ if "dir" in config.data_dir[data]:
+ input_dir = config.data_dir[data]["dir"]
+ data_type = config.data_dir[data]["type"]
+
+ if not os.path.exists(input_dir):
+ # create dummy data
+ print(f"-- Creating {n_files} dummy {data_type} files in {input_dir}")
+ os.makedirs(input_dir, exist_ok=True)
+ for i in range(n_files):
+ if data_type == "png":
+ dim = np.random.randint(100, 200)
+ img = np.random.randint(0, 255, (dim, dim, 3), dtype=np.uint8)
+ img_path = os.path.join(input_dir, f"{i}.png")
+ PILImage.fromarray(img).save(img_path)
+ elif data_type == "wav":
+ duration = np.random.randint(1, 4)
+ sample_rate = 16000
+ audio = np.random.randn(duration * sample_rate)
+ audio_path = os.path.join(input_dir, f"{i}.wav")
+ sf.write(audio_path, audio, samplerate=sample_rate)
+ elif data_type == "txt":
+ text = f"Hello, this is file {i}"
+ text_path = os.path.join(input_dir, f"{i}.txt")
+ with open(text_path, "w") as f:
+ f.write(text)
+
+ # check number of files
+ files = glob.glob(os.path.join(input_dir, "*." + data_type))
+ n_files = len(files)
+ print(f"Found {n_files} {data_type} files in {input_dir}")
+
+ # for CSV file where each line is a data point
+ elif "file" in config.data_dir[data]:
+ input_file = config.data_dir[data]["file"]
+
+ if not os.path.exists(input_file):
+ # create dummy labels
+ labels = ["good", "ok", "bad"]
+ file_labels = np.random.choice(labels, n_files)
+ with open(input_file, "w") as f:
+ for i in range(n_files):
+ f.write(f"{i},{file_labels[i]}\n")
+ print(f"-- Created dummy labels file at {input_file}")
+
+ # check number of unique labels (open with Pandas)
+ df = pd.read_csv(input_file, header=None)
+ n_files = len(df)
+ labels = df[1].unique()
+ n_labels = len(df[1].unique())
+ print(f"Found {n_files} lines with {n_labels} unique labels ({labels}) in {input_file}")
+
+ else:
+ raise ValueError("Please provide either `dir` or `file` in data_dir")
+
+ # -- only keep common files across all datasets
+ bn = [os.path.basename(f).split(".")[0] for f in files]
+ for data in config.data_dir:
+ if "dir" in config.data_dir[data]:
+ input_dir = config.data_dir[data]["dir"]
+ data_type = config.data_dir[data]["type"]
+ files = glob.glob(os.path.join(input_dir, "*." + data_type))
+ bn_data = [os.path.basename(f).split(".")[0] for f in files]
+ common_files = list(set(bn).intersection(bn_data))
+ common_files = natural_sort(common_files)
+ print(f"Number of common files: {len(common_files)}")
+
+ # -- add common files into dictionary
+ for data in config.data_dir:
+ if "dir" in config.data_dir[data]:
+ with open_dict(config):
+ config.data_dir[data]["data"] = common_files
+ if "file" in config.data_dir[data]:
+ # take row according to common_files
+ df = pd.read_csv(config.data_dir[data]["file"], header=None)
+ # -- make first column string
+ df[0] = df[0].astype(str)
+ df = df[df[0].isin(common_files)]
+ with open_dict(config):
+ config.data_dir[data]["data"] = df[1].tolist()
+
+ # Step 2: Create train and test data
+ dataset_dict = {}
+
+ # -- create dictionary of content
+ for data in config.data_dir:
+ if "dir" in config.data_dir[data]:
+ files = config.data_dir[data]["data"]
+ data_type = config.data_dir[data]["type"]
+ data_files = [
+ os.path.join(config.data_dir[data]["dir"], f"{f}.{data_type}") for f in files
+ ]
+
+ if data_type in ["txt"]:
+ # open file content for text files
+ data_files = [open(f).read() for f in data_files]
+ dataset_dict[data] = data_files
+ elif "file" in config.data_dir[data]:
+ dataset_dict[data] = config.data_dir[data]["data"]
+
+ # -- create dataset
+ dataset = Dataset.from_dict(dataset_dict)
+ for data in config.data_dir:
+ if "dir" in config.data_dir[data]:
+ if config.data_dir[data]["type"] in ["png", "jpg", "jpeg", "tiff"]:
+ dataset = dataset.cast_column(data, Image())
+ elif config.data_dir[data]["type"] in ["wav", "mp3", "flac", "ogg"]:
+ dataset = dataset.cast_column(data, Audio())
+ elif "file" in config.data_dir[data]:
+ if config.data_dir[data]["label"]:
+ labels = list(set(config.data_dir[data]["data"]))
+ dataset = dataset.cast_column(data, ClassLabel(names=labels))
+
+ # -- split into train and test
+ dataset = dataset.train_test_split(
+ test_size=test_size,
+ seed=config.seed,
+ shuffle=True,
+ stratify_by_column=config.stratify_by_column, # shuffle must be True
+ )
+ print(dataset)
+
+ """
+ DatasetDict({
+ train: Dataset({
+ features: ['audio', 'images', 'text', 'labels'],
+ num_rows: 85
+ })
+ test: Dataset({
+ features: ['audio', 'images', 'text', 'labels'],
+ num_rows: 15
+ })
+ })
+ """
+
+ # Step 3: Push to Hugging Face
+ dataset.push_to_hub(repo_id, token=hf_token)
+
+ # -- push individual files
+ for data in config.data_dir:
+ if "dir" in config.data_dir[data]:
+ # push first file
+ local_fp = os.path.join(
+ config.data_dir[data]["dir"],
+ config.data_dir[data]["data"][0] + "." + config.data_dir[data]["type"],
+ )
+ remote_fn = "example." + config.data_dir[data]["type"]
+ upload_file(
+ path_or_fileobj=local_fp,
+ path_in_repo=remote_fn,
+ repo_id=repo_id,
+ repo_type="dataset",
+ token=hf_token,
+ )
+
+ # total time in minutes
+ print(f"Total time: {(time.time() - start_time) / 60} minutes")
+
+
+if __name__ == "__main__":
+ main()

examples/use_huggingface_dataset.py

@@ -0,0 +1,50 @@
+"""
+In this script, we use the Hugging Face dataset made
+from the script examples/create_huggingface_dataset.py
+
+The dataset is available at:
+https://huggingface.co/datasets/bezzam/dummy-dataset
+
+```bash
+# install
+pip install datasets librosa soundfile
+
+# run
+python examples/use_huggingface_dataset.py
+```
+
+During the first run, the dataset will be downloaded and cached.
+Subsequent runs will use the cached dataset.
+
+"""
+
+from datasets import load_dataset
+import numpy as np
+
+
+# load train and test splits
+ds_train = load_dataset("bezzam/dummy-dataset", split="train")
+ds_test = load_dataset("bezzam/dummy-dataset", split="test")
+print(f"Number of training samples: {len(ds_train)}")
+print(f"Number of test samples: {len(ds_test)}")
+
+# load first example
+print("\n---- First example:")
+example = ds_train[0]
+
+# -- audio duration
+duration = len(example["audio"]["array"]) / example["audio"]["sampling_rate"]
+print(f"Duration of audio: {duration:.2f} seconds")
+
+# -- image size
+image = np.array(example["image"])
+print(f"Size of image: {image.shape}")
+
+# -- text
+text = example["text"]
+print(f"Text: {text}")
+
+# -- label
+label = example["label"]
+label_str = ds_train.features["label"].int2str(label)
+print(f"Label: {label_str}")

.gitignore

@@ -1,5 +1,6 @@
 project_env/
 .DS_Store
+dataset/*
 
 # from hydra
 outputs/ 

README.rst

@@ -61,7 +61,8 @@ choose the features that you like. This flexibility is one of the best
 * Code formatting.
 * Unit tests and continuous integration.
 * Packaging and distribution.
-* Remove development.
+* Remote development.
+* Creating and sharing datasets with Hugging Face.
 
 The accompanying 
 `slides <https://docs.google.com/presentation/d/1D1_JywMl2rjaeuVzpykPBOJsDIuwQKGOJB4EFZjej2s/edit#slide=id.g2eaa4b61f15_0_1346>`__ 

examples/configs/hf_dataset.yaml

@@ -0,0 +1,29 @@
+# python examples/create_huggingface_dataset.py hf_token=YOUR_TOKEN
+hydra:
+ job:
+ chdir: True # change to output folder
+ job_logging:
+ formatters:
+ simple:
+ format: '[%(levelname)s] - %(message)s'
+
+repo_id: bezzam/dummy-dataset
+seed: 0
+test_size: 0.15
+hf_token:
+
+data_dir:
+ audio:
+ dir: dataset/data_audio
+ type: wav
+ image:
+ dir: dataset/data_images
+ type: png
+ text:
+ dir: dataset/data_text
+ type: txt
+ label:
+ file: dataset/data_labels.csv
+ label: True
+
+stratify_by_column: label

examples/create_huggingface_dataset.py

@@ -0,0 +1,232 @@
+"""
+We will create a dataset with images, audios, and text data
+so that you can see how various data types can be pushed to
+Hugging Face!
+
+The default configuration is in `examples/configs/hf_dataset.yaml`:
+
+```bash
+# install
+pip install datasets huggingface_hub soundfile
+
+# make a WRITE token on HuggingFace: https://huggingface.co/settings/tokens
+
+# run
+python examples/create_huggingface_dataset.py \
+hf_token=... \
+```
+"""
+
+import hydra
+from hydra.utils import to_absolute_path
+import os
+import time
+import glob
+import numpy as np
+import soundfile as sf
+from PIL import Image as PILImage
+from datasets import Dataset, Image, Audio, ClassLabel
+from omegaconf import open_dict
+from huggingface_hub import upload_file
+import re
+import pandas as pd
+
+
+# -- helper functions
+def convert(text):
+ return int(text) if text.isdigit() else text.lower()
+
+
+def alphanum_key(key):
+ return [convert(c) for c in re.split("([0-9]+)", key)]
+
+
+def natural_sort(arr):
+ return sorted(arr, key=alphanum_key)
+
+
+@hydra.main(version_base=None, config_path="configs", config_name="hf_dataset")
+def main(config):
+
+ start_time = time.time()
+
+ # extract and check parameters
+ repo_id = config.repo_id
+ hf_token = config.hf_token
+ test_size = config.test_size
+
+ assert repo_id is not None, "Please provide a Hugging Face repo_id."
+ assert hf_token is not None, "Please provide a Hugging Face token."
+
+ # to absolute path, as needed by Hugging Face upload
+ for data in config.data_dir:
+ if "dir" in config.data_dir[data]:
+ config.data_dir[data]["dir"] = to_absolute_path(config.data_dir[data]["dir"])
+ elif "file" in config.data_dir[data]:
+ config.data_dir[data]["file"] = to_absolute_path(config.data_dir[data]["file"])
+
+ # Step 1: Check data (create dummy data if not present)
+ n_files = 100 # number of dummy files to create
+ for data in config.data_dir:
+
+ # for directory of data
+ if "dir" in config.data_dir[data]:
+ input_dir = config.data_dir[data]["dir"]
+ data_type = config.data_dir[data]["type"]
+
+ if not os.path.exists(input_dir):
+ # create dummy data
+ print(f"-- Creating {n_files} dummy {data_type} files in {input_dir}")
+ os.makedirs(input_dir, exist_ok=True)
+ for i in range(n_files):
+ if data_type == "png":
+ dim = np.random.randint(100, 200)
+ img = np.random.randint(0, 255, (dim, dim, 3), dtype=np.uint8)
+ img_path = os.path.join(input_dir, f"{i}.png")
+ PILImage.fromarray(img).save(img_path)
+ elif data_type == "wav":
+ duration = np.random.randint(1, 4)
+ sample_rate = 16000
+ audio = np.random.randn(duration * sample_rate)
+ audio_path = os.path.join(input_dir, f"{i}.wav")
+ sf.write(audio_path, audio, samplerate=sample_rate)
+ elif data_type == "txt":
+ text = f"Hello, this is file {i}"
+ text_path = os.path.join(input_dir, f"{i}.txt")
+ with open(text_path, "w") as f:
+ f.write(text)
+
+ # check number of files
+ files = glob.glob(os.path.join(input_dir, "*." + data_type))
+ n_files = len(files)
+ print(f"Found {n_files} {data_type} files in {input_dir}")
+
+ # for CSV file where each line is a data point
+ elif "file" in config.data_dir[data]:
+ input_file = config.data_dir[data]["file"]
+
+ if not os.path.exists(input_file):
+ # create dummy labels
+ labels = ["good", "ok", "bad"]
+ file_labels = np.random.choice(labels, n_files)
+ with open(input_file, "w") as f:
+ for i in range(n_files):
+ f.write(f"{i},{file_labels[i]}\n")
+ print(f"-- Created dummy labels file at {input_file}")
+
+ # check number of unique labels (open with Pandas)
+ df = pd.read_csv(input_file, header=None)
+ n_files = len(df)
+ labels = df[1].unique()
+ n_labels = len(df[1].unique())
+ print(f"Found {n_files} lines with {n_labels} unique labels ({labels}) in {input_file}")
+
+ else:
+ raise ValueError("Please provide either `dir` or `file` in data_dir")
+
+ # -- only keep common files across all datasets
+ bn = [os.path.basename(f).split(".")[0] for f in files]
+ for data in config.data_dir:
+ if "dir" in config.data_dir[data]:
+ input_dir = config.data_dir[data]["dir"]
+ data_type = config.data_dir[data]["type"]
+ files = glob.glob(os.path.join(input_dir, "*." + data_type))
+ bn_data = [os.path.basename(f).split(".")[0] for f in files]
+ common_files = list(set(bn).intersection(bn_data))
+ common_files = natural_sort(common_files)
+ print(f"Number of common files: {len(common_files)}")
+
+ # -- add common files into dictionary
+ for data in config.data_dir:
+ if "dir" in config.data_dir[data]:
+ with open_dict(config):
+ config.data_dir[data]["data"] = common_files
+ if "file" in config.data_dir[data]:
+ # take row according to common_files
+ df = pd.read_csv(config.data_dir[data]["file"], header=None)
+ # -- make first column string
+ df[0] = df[0].astype(str)
+ df = df[df[0].isin(common_files)]
+ with open_dict(config):
+ config.data_dir[data]["data"] = df[1].tolist()
+
+ # Step 2: Create train and test data
+ dataset_dict = {}
+
+ # -- create dictionary of content
+ for data in config.data_dir:
+ if "dir" in config.data_dir[data]:
+ files = config.data_dir[data]["data"]
+ data_type = config.data_dir[data]["type"]
+ data_files = [
+ os.path.join(config.data_dir[data]["dir"], f"{f}.{data_type}") for f in files
+ ]
+
+ if data_type in ["txt"]:
+ # open file content for text files
+ data_files = [open(f).read() for f in data_files]
+ dataset_dict[data] = data_files
+ elif "file" in config.data_dir[data]:
+ dataset_dict[data] = config.data_dir[data]["data"]
+
+ # -- create dataset
+ dataset = Dataset.from_dict(dataset_dict)
+ for data in config.data_dir:
+ if "dir" in config.data_dir[data]:
+ if config.data_dir[data]["type"] in ["png", "jpg", "jpeg", "tiff"]:
+ dataset = dataset.cast_column(data, Image())
+ elif config.data_dir[data]["type"] in ["wav", "mp3", "flac", "ogg"]:
+ dataset = dataset.cast_column(data, Audio())
+ elif "file" in config.data_dir[data]:
+ if config.data_dir[data]["label"]:
+ labels = list(set(config.data_dir[data]["data"]))
+ dataset = dataset.cast_column(data, ClassLabel(names=labels))
+
+ # -- split into train and test
+ dataset = dataset.train_test_split(
+ test_size=test_size,
+ seed=config.seed,
+ shuffle=True,
+ stratify_by_column=config.stratify_by_column, # shuffle must be True
+ )
+ print(dataset)
+
+ """
+ DatasetDict({
+ train: Dataset({
+ features: ['audio', 'images', 'text', 'labels'],
+ num_rows: 85
+ })
+ test: Dataset({
+ features: ['audio', 'images', 'text', 'labels'],
+ num_rows: 15
+ })
+ })
+ """
+
+ # Step 3: Push to Hugging Face
+ dataset.push_to_hub(repo_id, token=hf_token)
+
+ # -- push individual files
+ for data in config.data_dir:
+ if "dir" in config.data_dir[data]:
+ # push first file
+ local_fp = os.path.join(
+ config.data_dir[data]["dir"],
+ config.data_dir[data]["data"][0] + "." + config.data_dir[data]["type"],
+ )
+ remote_fn = "example." + config.data_dir[data]["type"]
+ upload_file(
+ path_or_fileobj=local_fp,
+ path_in_repo=remote_fn,
+ repo_id=repo_id,
+ repo_type="dataset",
+ token=hf_token,
+ )
+
+ # total time in minutes
+ print(f"Total time: {(time.time() - start_time) / 60} minutes")
+
+
+if __name__ == "__main__":
+ main()

examples/use_huggingface_dataset.py

@@ -0,0 +1,50 @@
+"""
+In this script, we use the Hugging Face dataset made
+from the script examples/create_huggingface_dataset.py
+
+The dataset is available at:
+https://huggingface.co/datasets/bezzam/dummy-dataset
+
+```bash
+# install
+pip install datasets librosa soundfile
+
+# run
+python examples/use_huggingface_dataset.py
+```
+
+During the first run, the dataset will be downloaded and cached.
+Subsequent runs will use the cached dataset.
+
+"""
+
+from datasets import load_dataset
+import numpy as np
+
+
+# load train and test splits
+ds_train = load_dataset("bezzam/dummy-dataset", split="train")
+ds_test = load_dataset("bezzam/dummy-dataset", split="test")
+print(f"Number of training samples: {len(ds_train)}")
+print(f"Number of test samples: {len(ds_test)}")
+
+# load first example
+print("\n---- First example:")
+example = ds_train[0]
+
+# -- audio duration
+duration = len(example["audio"]["array"]) / example["audio"]["sampling_rate"]
+print(f"Duration of audio: {duration:.2f} seconds")
+
+# -- image size
+image = np.array(example["image"])
+print(f"Size of image: {image.shape}")
+
+# -- text
+text = example["text"]
+print(f"Text: {text}")
+
+# -- label
+label = example["label"]
+label_str = ds_train.features["label"].int2str(label)
+print(f"Label: {label_str}")

.gitignore

@@ -1,5 +1,6 @@
 project_env/
 .DS_Store
+dataset/*
 
 # from hydra
 outputs/ 

README.rst

@@ -61,7 +61,8 @@ choose the features that you like. This flexibility is one of the best
 * Code formatting.
 * Unit tests and continuous integration.
 * Packaging and distribution.
-* Remove development.
+* Remote development.
+* Creating and sharing datasets with Hugging Face.
 
 The accompanying 
 `slides <https://docs.google.com/presentation/d/1D1_JywMl2rjaeuVzpykPBOJsDIuwQKGOJB4EFZjej2s/edit#slide=id.g2eaa4b61f15_0_1346>`__ 

examples/configs/hf_dataset.yaml

@@ -0,0 +1,29 @@
+# python examples/create_huggingface_dataset.py hf_token=YOUR_TOKEN
+hydra:
+ job:
+ chdir: True # change to output folder
+ job_logging:
+ formatters:
+ simple:
+ format: '[%(levelname)s] - %(message)s'
+
+repo_id: bezzam/dummy-dataset
+seed: 0
+test_size: 0.15
+hf_token:
+
+data_dir:
+ audio:
+ dir: dataset/data_audio
+ type: wav
+ image:
+ dir: dataset/data_images
+ type: png
+ text:
+ dir: dataset/data_text
+ type: txt
+ label:
+ file: dataset/data_labels.csv
+ label: True
+
+stratify_by_column: label

examples/create_huggingface_dataset.py

@@ -0,0 +1,232 @@
+"""
+We will create a dataset with images, audios, and text data
+so that you can see how various data types can be pushed to
+Hugging Face!
+
+The default configuration is in `examples/configs/hf_dataset.yaml`:
+
+```bash
+# install
+pip install datasets huggingface_hub soundfile
+
+# make a WRITE token on HuggingFace: https://huggingface.co/settings/tokens
+
+# run
+python examples/create_huggingface_dataset.py \
+hf_token=... \
+```
+"""
+
+import hydra
+from hydra.utils import to_absolute_path
+import os
+import time
+import glob
+import numpy as np
+import soundfile as sf
+from PIL import Image as PILImage
+from datasets import Dataset, Image, Audio, ClassLabel
+from omegaconf import open_dict
+from huggingface_hub import upload_file
+import re
+import pandas as pd
+
+
+# -- helper functions
+def convert(text):
+ return int(text) if text.isdigit() else text.lower()
+
+
+def alphanum_key(key):
+ return [convert(c) for c in re.split("([0-9]+)", key)]
+
+
+def natural_sort(arr):
+ return sorted(arr, key=alphanum_key)
+
+
+@hydra.main(version_base=None, config_path="configs", config_name="hf_dataset")
+def main(config):
+
+ start_time = time.time()
+
+ # extract and check parameters
+ repo_id = config.repo_id
+ hf_token = config.hf_token
+ test_size = config.test_size
+
+ assert repo_id is not None, "Please provide a Hugging Face repo_id."
+ assert hf_token is not None, "Please provide a Hugging Face token."
+
+ # to absolute path, as needed by Hugging Face upload
+ for data in config.data_dir:
+ if "dir" in config.data_dir[data]:
+ config.data_dir[data]["dir"] = to_absolute_path(config.data_dir[data]["dir"])
+ elif "file" in config.data_dir[data]:
+ config.data_dir[data]["file"] = to_absolute_path(config.data_dir[data]["file"])
+
+ # Step 1: Check data (create dummy data if not present)
+ n_files = 100 # number of dummy files to create
+ for data in config.data_dir:
+
+ # for directory of data
+ if "dir" in config.data_dir[data]:
+ input_dir = config.data_dir[data]["dir"]
+ data_type = config.data_dir[data]["type"]
+
+ if not os.path.exists(input_dir):
+ # create dummy data
+ print(f"-- Creating {n_files} dummy {data_type} files in {input_dir}")
+ os.makedirs(input_dir, exist_ok=True)
+ for i in range(n_files):
+ if data_type == "png":
+ dim = np.random.randint(100, 200)
+ img = np.random.randint(0, 255, (dim, dim, 3), dtype=np.uint8)
+ img_path = os.path.join(input_dir, f"{i}.png")
+ PILImage.fromarray(img).save(img_path)
+ elif data_type == "wav":
+ duration = np.random.randint(1, 4)
+ sample_rate = 16000
+ audio = np.random.randn(duration * sample_rate)
+ audio_path = os.path.join(input_dir, f"{i}.wav")
+ sf.write(audio_path, audio, samplerate=sample_rate)
+ elif data_type == "txt":
+ text = f"Hello, this is file {i}"
+ text_path = os.path.join(input_dir, f"{i}.txt")
+ with open(text_path, "w") as f:
+ f.write(text)
+
+ # check number of files
+ files = glob.glob(os.path.join(input_dir, "*." + data_type))
+ n_files = len(files)
+ print(f"Found {n_files} {data_type} files in {input_dir}")
+
+ # for CSV file where each line is a data point
+ elif "file" in config.data_dir[data]:
+ input_file = config.data_dir[data]["file"]
+
+ if not os.path.exists(input_file):
+ # create dummy labels
+ labels = ["good", "ok", "bad"]
+ file_labels = np.random.choice(labels, n_files)
+ with open(input_file, "w") as f:
+ for i in range(n_files):
+ f.write(f"{i},{file_labels[i]}\n")
+ print(f"-- Created dummy labels file at {input_file}")
+
+ # check number of unique labels (open with Pandas)
+ df = pd.read_csv(input_file, header=None)
+ n_files = len(df)
+ labels = df[1].unique()
+ n_labels = len(df[1].unique())
+ print(f"Found {n_files} lines with {n_labels} unique labels ({labels}) in {input_file}")
+
+ else:
+ raise ValueError("Please provide either `dir` or `file` in data_dir")
+
+ # -- only keep common files across all datasets
+ bn = [os.path.basename(f).split(".")[0] for f in files]
+ for data in config.data_dir:
+ if "dir" in config.data_dir[data]:
+ input_dir = config.data_dir[data]["dir"]
+ data_type = config.data_dir[data]["type"]
+ files = glob.glob(os.path.join(input_dir, "*." + data_type))
+ bn_data = [os.path.basename(f).split(".")[0] for f in files]
+ common_files = list(set(bn).intersection(bn_data))
+ common_files = natural_sort(common_files)
+ print(f"Number of common files: {len(common_files)}")
+
+ # -- add common files into dictionary
+ for data in config.data_dir:
+ if "dir" in config.data_dir[data]:
+ with open_dict(config):
+ config.data_dir[data]["data"] = common_files
+ if "file" in config.data_dir[data]:
+ # take row according to common_files
+ df = pd.read_csv(config.data_dir[data]["file"], header=None)
+ # -- make first column string
+ df[0] = df[0].astype(str)
+ df = df[df[0].isin(common_files)]
+ with open_dict(config):
+ config.data_dir[data]["data"] = df[1].tolist()
+
+ # Step 2: Create train and test data
+ dataset_dict = {}
+
+ # -- create dictionary of content
+ for data in config.data_dir:
+ if "dir" in config.data_dir[data]:
+ files = config.data_dir[data]["data"]
+ data_type = config.data_dir[data]["type"]
+ data_files = [
+ os.path.join(config.data_dir[data]["dir"], f"{f}.{data_type}") for f in files
+ ]
+
+ if data_type in ["txt"]:
+ # open file content for text files
+ data_files = [open(f).read() for f in data_files]
+ dataset_dict[data] = data_files
+ elif "file" in config.data_dir[data]:
+ dataset_dict[data] = config.data_dir[data]["data"]
+
+ # -- create dataset
+ dataset = Dataset.from_dict(dataset_dict)
+ for data in config.data_dir:
+ if "dir" in config.data_dir[data]:
+ if config.data_dir[data]["type"] in ["png", "jpg", "jpeg", "tiff"]:
+ dataset = dataset.cast_column(data, Image())
+ elif config.data_dir[data]["type"] in ["wav", "mp3", "flac", "ogg"]:
+ dataset = dataset.cast_column(data, Audio())
+ elif "file" in config.data_dir[data]:
+ if config.data_dir[data]["label"]:
+ labels = list(set(config.data_dir[data]["data"]))
+ dataset = dataset.cast_column(data, ClassLabel(names=labels))
+
+ # -- split into train and test
+ dataset = dataset.train_test_split(
+ test_size=test_size,
+ seed=config.seed,
+ shuffle=True,
+ stratify_by_column=config.stratify_by_column, # shuffle must be True
+ )
+ print(dataset)
+
+ """
+ DatasetDict({
+ train: Dataset({
+ features: ['audio', 'images', 'text', 'labels'],
+ num_rows: 85
+ })
+ test: Dataset({
+ features: ['audio', 'images', 'text', 'labels'],
+ num_rows: 15
+ })
+ })
+ """
+
+ # Step 3: Push to Hugging Face
+ dataset.push_to_hub(repo_id, token=hf_token)
+
+ # -- push individual files
+ for data in config.data_dir:
+ if "dir" in config.data_dir[data]:
+ # push first file
+ local_fp = os.path.join(
+ config.data_dir[data]["dir"],
+ config.data_dir[data]["data"][0] + "." + config.data_dir[data]["type"],
+ )
+ remote_fn = "example." + config.data_dir[data]["type"]
+ upload_file(
+ path_or_fileobj=local_fp,
+ path_in_repo=remote_fn,
+ repo_id=repo_id,
+ repo_type="dataset",
+ token=hf_token,
+ )
+
+ # total time in minutes
+ print(f"Total time: {(time.time() - start_time) / 60} minutes")
+
+
+if __name__ == "__main__":
+ main()

examples/use_huggingface_dataset.py

@@ -0,0 +1,50 @@
+"""
+In this script, we use the Hugging Face dataset made
+from the script examples/create_huggingface_dataset.py
+
+The dataset is available at:
+https://huggingface.co/datasets/bezzam/dummy-dataset
+
+```bash
+# install
+pip install datasets librosa soundfile
+
+# run
+python examples/use_huggingface_dataset.py
+```
+
+During the first run, the dataset will be downloaded and cached.
+Subsequent runs will use the cached dataset.
+
+"""
+
+from datasets import load_dataset
+import numpy as np
+
+
+# load train and test splits
+ds_train = load_dataset("bezzam/dummy-dataset", split="train")
+ds_test = load_dataset("bezzam/dummy-dataset", split="test")
+print(f"Number of training samples: {len(ds_train)}")
+print(f"Number of test samples: {len(ds_test)}")
+
+# load first example
+print("\n---- First example:")
+example = ds_train[0]
+
+# -- audio duration
+duration = len(example["audio"]["array"]) / example["audio"]["sampling_rate"]
+print(f"Duration of audio: {duration:.2f} seconds")
+
+# -- image size
+image = np.array(example["image"])
+print(f"Size of image: {image.shape}")
+
+# -- text
+text = example["text"]
+print(f"Text: {text}")
+
+# -- label
+label = example["label"]
+label_str = ds_train.features["label"].int2str(label)
+print(f"Label: {label_str}")

.gitignore

@@ -1,5 +1,6 @@
 project_env/
 .DS_Store
+dataset/*
 
 # from hydra
 outputs/ 

README.rst

@@ -61,7 +61,8 @@ choose the features that you like. This flexibility is one of the best
 * Code formatting.
 * Unit tests and continuous integration.
 * Packaging and distribution.
-* Remove development.
+* Remote development.
+* Creating and sharing datasets with Hugging Face.
 
 The accompanying 
 `slides <https://docs.google.com/presentation/d/1D1_JywMl2rjaeuVzpykPBOJsDIuwQKGOJB4EFZjej2s/edit#slide=id.g2eaa4b61f15_0_1346>`__ 

examples/configs/hf_dataset.yaml

@@ -0,0 +1,29 @@
+# python examples/create_huggingface_dataset.py hf_token=YOUR_TOKEN
+hydra:
+ job:
+ chdir: True # change to output folder
+ job_logging:
+ formatters:
+ simple:
+ format: '[%(levelname)s] - %(message)s'
+
+repo_id: bezzam/dummy-dataset
+seed: 0
+test_size: 0.15
+hf_token:
+
+data_dir:
+ audio:
+ dir: dataset/data_audio
+ type: wav
+ image:
+ dir: dataset/data_images
+ type: png
+ text:
+ dir: dataset/data_text
+ type: txt
+ label:
+ file: dataset/data_labels.csv
+ label: True
+
+stratify_by_column: label

examples/create_huggingface_dataset.py

@@ -0,0 +1,232 @@
+"""
+We will create a dataset with images, audios, and text data
+so that you can see how various data types can be pushed to
+Hugging Face!
+
+The default configuration is in `examples/configs/hf_dataset.yaml`:
+
+```bash
+# install
+pip install datasets huggingface_hub soundfile
+
+# make a WRITE token on HuggingFace: https://huggingface.co/settings/tokens
+
+# run
+python examples/create_huggingface_dataset.py \
+hf_token=... \
+```
+"""
+
+import hydra
+from hydra.utils import to_absolute_path
+import os
+import time
+import glob
+import numpy as np
+import soundfile as sf
+from PIL import Image as PILImage
+from datasets import Dataset, Image, Audio, ClassLabel
+from omegaconf import open_dict
+from huggingface_hub import upload_file
+import re
+import pandas as pd
+
+
+# -- helper functions
+def convert(text):
+ return int(text) if text.isdigit() else text.lower()
+
+
+def alphanum_key(key):
+ return [convert(c) for c in re.split("([0-9]+)", key)]
+
+
+def natural_sort(arr):
+ return sorted(arr, key=alphanum_key)
+
+
+@hydra.main(version_base=None, config_path="configs", config_name="hf_dataset")
+def main(config):
+
+ start_time = time.time()
+
+ # extract and check parameters
+ repo_id = config.repo_id
+ hf_token = config.hf_token
+ test_size = config.test_size
+
+ assert repo_id is not None, "Please provide a Hugging Face repo_id."
+ assert hf_token is not None, "Please provide a Hugging Face token."
+
+ # to absolute path, as needed by Hugging Face upload
+ for data in config.data_dir:
+ if "dir" in config.data_dir[data]:
+ config.data_dir[data]["dir"] = to_absolute_path(config.data_dir[data]["dir"])
+ elif "file" in config.data_dir[data]:
+ config.data_dir[data]["file"] = to_absolute_path(config.data_dir[data]["file"])
+
+ # Step 1: Check data (create dummy data if not present)
+ n_files = 100 # number of dummy files to create
+ for data in config.data_dir:
+
+ # for directory of data
+ if "dir" in config.data_dir[data]:
+ input_dir = config.data_dir[data]["dir"]
+ data_type = config.data_dir[data]["type"]
+
+ if not os.path.exists(input_dir):
+ # create dummy data
+ print(f"-- Creating {n_files} dummy {data_type} files in {input_dir}")
+ os.makedirs(input_dir, exist_ok=True)
+ for i in range(n_files):
+ if data_type == "png":
+ dim = np.random.randint(100, 200)
+ img = np.random.randint(0, 255, (dim, dim, 3), dtype=np.uint8)
+ img_path = os.path.join(input_dir, f"{i}.png")
+ PILImage.fromarray(img).save(img_path)
+ elif data_type == "wav":
+ duration = np.random.randint(1, 4)
+ sample_rate = 16000
+ audio = np.random.randn(duration * sample_rate)
+ audio_path = os.path.join(input_dir, f"{i}.wav")
+ sf.write(audio_path, audio, samplerate=sample_rate)
+ elif data_type == "txt":
+ text = f"Hello, this is file {i}"
+ text_path = os.path.join(input_dir, f"{i}.txt")
+ with open(text_path, "w") as f:
+ f.write(text)
+
+ # check number of files
+ files = glob.glob(os.path.join(input_dir, "*." + data_type))
+ n_files = len(files)
+ print(f"Found {n_files} {data_type} files in {input_dir}")
+
+ # for CSV file where each line is a data point
+ elif "file" in config.data_dir[data]:
+ input_file = config.data_dir[data]["file"]
+
+ if not os.path.exists(input_file):
+ # create dummy labels
+ labels = ["good", "ok", "bad"]
+ file_labels = np.random.choice(labels, n_files)
+ with open(input_file, "w") as f:
+ for i in range(n_files):
+ f.write(f"{i},{file_labels[i]}\n")
+ print(f"-- Created dummy labels file at {input_file}")
+
+ # check number of unique labels (open with Pandas)
+ df = pd.read_csv(input_file, header=None)
+ n_files = len(df)
+ labels = df[1].unique()
+ n_labels = len(df[1].unique())
+ print(f"Found {n_files} lines with {n_labels} unique labels ({labels}) in {input_file}")
+
+ else:
+ raise ValueError("Please provide either `dir` or `file` in data_dir")
+
+ # -- only keep common files across all datasets
+ bn = [os.path.basename(f).split(".")[0] for f in files]
+ for data in config.data_dir:
+ if "dir" in config.data_dir[data]:
+ input_dir = config.data_dir[data]["dir"]
+ data_type = config.data_dir[data]["type"]
+ files = glob.glob(os.path.join(input_dir, "*." + data_type))
+ bn_data = [os.path.basename(f).split(".")[0] for f in files]
+ common_files = list(set(bn).intersection(bn_data))
+ common_files = natural_sort(common_files)
+ print(f"Number of common files: {len(common_files)}")
+
+ # -- add common files into dictionary
+ for data in config.data_dir:
+ if "dir" in config.data_dir[data]:
+ with open_dict(config):
+ config.data_dir[data]["data"] = common_files
+ if "file" in config.data_dir[data]:
+ # take row according to common_files
+ df = pd.read_csv(config.data_dir[data]["file"], header=None)
+ # -- make first column string
+ df[0] = df[0].astype(str)
+ df = df[df[0].isin(common_files)]
+ with open_dict(config):
+ config.data_dir[data]["data"] = df[1].tolist()
+
+ # Step 2: Create train and test data
+ dataset_dict = {}
+
+ # -- create dictionary of content
+ for data in config.data_dir:
+ if "dir" in config.data_dir[data]:
+ files = config.data_dir[data]["data"]
+ data_type = config.data_dir[data]["type"]
+ data_files = [
+ os.path.join(config.data_dir[data]["dir"], f"{f}.{data_type}") for f in files
+ ]
+
+ if data_type in ["txt"]:
+ # open file content for text files
+ data_files = [open(f).read() for f in data_files]
+ dataset_dict[data] = data_files
+ elif "file" in config.data_dir[data]:
+ dataset_dict[data] = config.data_dir[data]["data"]
+
+ # -- create dataset
+ dataset = Dataset.from_dict(dataset_dict)
+ for data in config.data_dir:
+ if "dir" in config.data_dir[data]:
+ if config.data_dir[data]["type"] in ["png", "jpg", "jpeg", "tiff"]:
+ dataset = dataset.cast_column(data, Image())
+ elif config.data_dir[data]["type"] in ["wav", "mp3", "flac", "ogg"]:
+ dataset = dataset.cast_column(data, Audio())
+ elif "file" in config.data_dir[data]:
+ if config.data_dir[data]["label"]:
+ labels = list(set(config.data_dir[data]["data"]))
+ dataset = dataset.cast_column(data, ClassLabel(names=labels))
+
+ # -- split into train and test
+ dataset = dataset.train_test_split(
+ test_size=test_size,
+ seed=config.seed,
+ shuffle=True,
+ stratify_by_column=config.stratify_by_column, # shuffle must be True
+ )
+ print(dataset)
+
+ """
+ DatasetDict({
+ train: Dataset({
+ features: ['audio', 'images', 'text', 'labels'],
+ num_rows: 85
+ })
+ test: Dataset({
+ features: ['audio', 'images', 'text', 'labels'],
+ num_rows: 15
+ })
+ })
+ """
+
+ # Step 3: Push to Hugging Face
+ dataset.push_to_hub(repo_id, token=hf_token)
+
+ # -- push individual files
+ for data in config.data_dir:
+ if "dir" in config.data_dir[data]:
+ # push first file
+ local_fp = os.path.join(
+ config.data_dir[data]["dir"],
+ config.data_dir[data]["data"][0] + "." + config.data_dir[data]["type"],
+ )
+ remote_fn = "example." + config.data_dir[data]["type"]
+ upload_file(
+ path_or_fileobj=local_fp,
+ path_in_repo=remote_fn,
+ repo_id=repo_id,
+ repo_type="dataset",
+ token=hf_token,
+ )
+
+ # total time in minutes
+ print(f"Total time: {(time.time() - start_time) / 60} minutes")
+
+
+if __name__ == "__main__":
+ main()

examples/use_huggingface_dataset.py

@@ -0,0 +1,50 @@
+"""
+In this script, we use the Hugging Face dataset made
+from the script examples/create_huggingface_dataset.py
+
+The dataset is available at:
+https://huggingface.co/datasets/bezzam/dummy-dataset
+
+```bash
+# install
+pip install datasets librosa soundfile
+
+# run
+python examples/use_huggingface_dataset.py
+```
+
+During the first run, the dataset will be downloaded and cached.
+Subsequent runs will use the cached dataset.
+
+"""
+
+from datasets import load_dataset
+import numpy as np
+
+
+# load train and test splits
+ds_train = load_dataset("bezzam/dummy-dataset", split="train")
+ds_test = load_dataset("bezzam/dummy-dataset", split="test")
+print(f"Number of training samples: {len(ds_train)}")
+print(f"Number of test samples: {len(ds_test)}")
+
+# load first example
+print("\n---- First example:")
+example = ds_train[0]
+
+# -- audio duration
+duration = len(example["audio"]["array"]) / example["audio"]["sampling_rate"]
+print(f"Duration of audio: {duration:.2f} seconds")
+
+# -- image size
+image = np.array(example["image"])
+print(f"Size of image: {image.shape}")
+
+# -- text
+text = example["text"]
+print(f"Text: {text}")
+
+# -- label
+label = example["label"]
+label_str = ds_train.features["label"].int2str(label)
+print(f"Label: {label_str}")

.gitignore

@@ -1,5 +1,6 @@
 project_env/
 .DS_Store
+dataset/*
 
 # from hydra
 outputs/ 

README.rst

@@ -61,7 +61,8 @@ choose the features that you like. This flexibility is one of the best
 * Code formatting.
 * Unit tests and continuous integration.
 * Packaging and distribution.
-* Remove development.
+* Remote development.
+* Creating and sharing datasets with Hugging Face.
 
 The accompanying 
 `slides <https://docs.google.com/presentation/d/1D1_JywMl2rjaeuVzpykPBOJsDIuwQKGOJB4EFZjej2s/edit#slide=id.g2eaa4b61f15_0_1346>`__ 

examples/configs/hf_dataset.yaml

@@ -0,0 +1,29 @@
+# python examples/create_huggingface_dataset.py hf_token=YOUR_TOKEN
+hydra:
+ job:
+ chdir: True # change to output folder
+ job_logging:
+ formatters:
+ simple:
+ format: '[%(levelname)s] - %(message)s'
+
+repo_id: bezzam/dummy-dataset
+seed: 0
+test_size: 0.15
+hf_token:
+
+data_dir:
+ audio:
+ dir: dataset/data_audio
+ type: wav
+ image:
+ dir: dataset/data_images
+ type: png
+ text:
+ dir: dataset/data_text
+ type: txt
+ label:
+ file: dataset/data_labels.csv
+ label: True
+
+stratify_by_column: label

examples/create_huggingface_dataset.py

@@ -0,0 +1,232 @@
+"""
+We will create a dataset with images, audios, and text data
+so that you can see how various data types can be pushed to
+Hugging Face!
+
+The default configuration is in `examples/configs/hf_dataset.yaml`:
+
+```bash
+# install
+pip install datasets huggingface_hub soundfile
+
+# make a WRITE token on HuggingFace: https://huggingface.co/settings/tokens
+
+# run
+python examples/create_huggingface_dataset.py \
+hf_token=... \
+```
+"""
+
+import hydra
+from hydra.utils import to_absolute_path
+import os
+import time
+import glob
+import numpy as np
+import soundfile as sf
+from PIL import Image as PILImage
+from datasets import Dataset, Image, Audio, ClassLabel
+from omegaconf import open_dict
+from huggingface_hub import upload_file
+import re
+import pandas as pd
+
+
+# -- helper functions
+def convert(text):
+ return int(text) if text.isdigit() else text.lower()
+
+
+def alphanum_key(key):
+ return [convert(c) for c in re.split("([0-9]+)", key)]
+
+
+def natural_sort(arr):
+ return sorted(arr, key=alphanum_key)
+
+
+@hydra.main(version_base=None, config_path="configs", config_name="hf_dataset")
+def main(config):
+
+ start_time = time.time()
+
+ # extract and check parameters
+ repo_id = config.repo_id
+ hf_token = config.hf_token
+ test_size = config.test_size
+
+ assert repo_id is not None, "Please provide a Hugging Face repo_id."
+ assert hf_token is not None, "Please provide a Hugging Face token."
+
+ # to absolute path, as needed by Hugging Face upload
+ for data in config.data_dir:
+ if "dir" in config.data_dir[data]:
+ config.data_dir[data]["dir"] = to_absolute_path(config.data_dir[data]["dir"])
+ elif "file" in config.data_dir[data]:
+ config.data_dir[data]["file"] = to_absolute_path(config.data_dir[data]["file"])
+
+ # Step 1: Check data (create dummy data if not present)
+ n_files = 100 # number of dummy files to create
+ for data in config.data_dir:
+
+ # for directory of data
+ if "dir" in config.data_dir[data]:
+ input_dir = config.data_dir[data]["dir"]
+ data_type = config.data_dir[data]["type"]
+
+ if not os.path.exists(input_dir):
+ # create dummy data
+ print(f"-- Creating {n_files} dummy {data_type} files in {input_dir}")
+ os.makedirs(input_dir, exist_ok=True)
+ for i in range(n_files):
+ if data_type == "png":
+ dim = np.random.randint(100, 200)
+ img = np.random.randint(0, 255, (dim, dim, 3), dtype=np.uint8)
+ img_path = os.path.join(input_dir, f"{i}.png")
+ PILImage.fromarray(img).save(img_path)
+ elif data_type == "wav":
+ duration = np.random.randint(1, 4)
+ sample_rate = 16000
+ audio = np.random.randn(duration * sample_rate)
+ audio_path = os.path.join(input_dir, f"{i}.wav")
+ sf.write(audio_path, audio, samplerate=sample_rate)
+ elif data_type == "txt":
+ text = f"Hello, this is file {i}"
+ text_path = os.path.join(input_dir, f"{i}.txt")
+ with open(text_path, "w") as f:
+ f.write(text)
+
+ # check number of files
+ files = glob.glob(os.path.join(input_dir, "*." + data_type))
+ n_files = len(files)
+ print(f"Found {n_files} {data_type} files in {input_dir}")
+
+ # for CSV file where each line is a data point
+ elif "file" in config.data_dir[data]:
+ input_file = config.data_dir[data]["file"]
+
+ if not os.path.exists(input_file):
+ # create dummy labels
+ labels = ["good", "ok", "bad"]
+ file_labels = np.random.choice(labels, n_files)
+ with open(input_file, "w") as f:
+ for i in range(n_files):
+ f.write(f"{i},{file_labels[i]}\n")
+ print(f"-- Created dummy labels file at {input_file}")
+
+ # check number of unique labels (open with Pandas)
+ df = pd.read_csv(input_file, header=None)
+ n_files = len(df)
+ labels = df[1].unique()
+ n_labels = len(df[1].unique())
+ print(f"Found {n_files} lines with {n_labels} unique labels ({labels}) in {input_file}")
+
+ else:
+ raise ValueError("Please provide either `dir` or `file` in data_dir")
+
+ # -- only keep common files across all datasets
+ bn = [os.path.basename(f).split(".")[0] for f in files]
+ for data in config.data_dir:
+ if "dir" in config.data_dir[data]:
+ input_dir = config.data_dir[data]["dir"]
+ data_type = config.data_dir[data]["type"]
+ files = glob.glob(os.path.join(input_dir, "*." + data_type))
+ bn_data = [os.path.basename(f).split(".")[0] for f in files]
+ common_files = list(set(bn).intersection(bn_data))
+ common_files = natural_sort(common_files)
+ print(f"Number of common files: {len(common_files)}")
+
+ # -- add common files into dictionary
+ for data in config.data_dir:
+ if "dir" in config.data_dir[data]:
+ with open_dict(config):
+ config.data_dir[data]["data"] = common_files
+ if "file" in config.data_dir[data]:
+ # take row according to common_files
+ df = pd.read_csv(config.data_dir[data]["file"], header=None)
+ # -- make first column string
+ df[0] = df[0].astype(str)
+ df = df[df[0].isin(common_files)]
+ with open_dict(config):
+ config.data_dir[data]["data"] = df[1].tolist()
+
+ # Step 2: Create train and test data
+ dataset_dict = {}
+
+ # -- create dictionary of content
+ for data in config.data_dir:
+ if "dir" in config.data_dir[data]:
+ files = config.data_dir[data]["data"]
+ data_type = config.data_dir[data]["type"]
+ data_files = [
+ os.path.join(config.data_dir[data]["dir"], f"{f}.{data_type}") for f in files
+ ]
+
+ if data_type in ["txt"]:
+ # open file content for text files
+ data_files = [open(f).read() for f in data_files]
+ dataset_dict[data] = data_files
+ elif "file" in config.data_dir[data]:
+ dataset_dict[data] = config.data_dir[data]["data"]
+
+ # -- create dataset
+ dataset = Dataset.from_dict(dataset_dict)
+ for data in config.data_dir:
+ if "dir" in config.data_dir[data]:
+ if config.data_dir[data]["type"] in ["png", "jpg", "jpeg", "tiff"]:
+ dataset = dataset.cast_column(data, Image())
+ elif config.data_dir[data]["type"] in ["wav", "mp3", "flac", "ogg"]:
+ dataset = dataset.cast_column(data, Audio())
+ elif "file" in config.data_dir[data]:
+ if config.data_dir[data]["label"]:
+ labels = list(set(config.data_dir[data]["data"]))
+ dataset = dataset.cast_column(data, ClassLabel(names=labels))
+
+ # -- split into train and test
+ dataset = dataset.train_test_split(
+ test_size=test_size,
+ seed=config.seed,
+ shuffle=True,
+ stratify_by_column=config.stratify_by_column, # shuffle must be True
+ )
+ print(dataset)
+
+ """
+ DatasetDict({
+ train: Dataset({
+ features: ['audio', 'images', 'text', 'labels'],
+ num_rows: 85
+ })
+ test: Dataset({
+ features: ['audio', 'images', 'text', 'labels'],
+ num_rows: 15
+ })
+ })
+ """
+
+ # Step 3: Push to Hugging Face
+ dataset.push_to_hub(repo_id, token=hf_token)
+
+ # -- push individual files
+ for data in config.data_dir:
+ if "dir" in config.data_dir[data]:
+ # push first file
+ local_fp = os.path.join(
+ config.data_dir[data]["dir"],
+ config.data_dir[data]["data"][0] + "." + config.data_dir[data]["type"],
+ )
+ remote_fn = "example." + config.data_dir[data]["type"]
+ upload_file(
+ path_or_fileobj=local_fp,
+ path_in_repo=remote_fn,
+ repo_id=repo_id,
+ repo_type="dataset",
+ token=hf_token,
+ )
+
+ # total time in minutes
+ print(f"Total time: {(time.time() - start_time) / 60} minutes")
+
+
+if __name__ == "__main__":
+ main()

examples/use_huggingface_dataset.py

@@ -0,0 +1,50 @@
+"""
+In this script, we use the Hugging Face dataset made
+from the script examples/create_huggingface_dataset.py
+
+The dataset is available at:
+https://huggingface.co/datasets/bezzam/dummy-dataset
+
+```bash
+# install
+pip install datasets librosa soundfile
+
+# run
+python examples/use_huggingface_dataset.py
+```
+
+During the first run, the dataset will be downloaded and cached.
+Subsequent runs will use the cached dataset.
+
+"""
+
+from datasets import load_dataset
+import numpy as np
+
+
+# load train and test splits
+ds_train = load_dataset("bezzam/dummy-dataset", split="train")
+ds_test = load_dataset("bezzam/dummy-dataset", split="test")
+print(f"Number of training samples: {len(ds_train)}")
+print(f"Number of test samples: {len(ds_test)}")
+
+# load first example
+print("\n---- First example:")
+example = ds_train[0]
+
+# -- audio duration
+duration = len(example["audio"]["array"]) / example["audio"]["sampling_rate"]
+print(f"Duration of audio: {duration:.2f} seconds")
+
+# -- image size
+image = np.array(example["image"])
+print(f"Size of image: {image.shape}")
+
+# -- text
+text = example["text"]
+print(f"Text: {text}")
+
+# -- label
+label = example["label"]
+label_str = ds_train.features["label"].int2str(label)
+print(f"Label: {label_str}")

.gitignore

@@ -1,5 +1,6 @@
 project_env/
 .DS_Store
+dataset/*
 
 # from hydra
 outputs/ 

README.rst

@@ -61,7 +61,8 @@ choose the features that you like. This flexibility is one of the best
 * Code formatting.
 * Unit tests and continuous integration.
 * Packaging and distribution.
-* Remove development.
+* Remote development.
+* Creating and sharing datasets with Hugging Face.
 
 The accompanying 
 `slides <https://docs.google.com/presentation/d/1D1_JywMl2rjaeuVzpykPBOJsDIuwQKGOJB4EFZjej2s/edit#slide=id.g2eaa4b61f15_0_1346>`__ 

examples/configs/hf_dataset.yaml

@@ -0,0 +1,29 @@
+# python examples/create_huggingface_dataset.py hf_token=YOUR_TOKEN
+hydra:
+ job:
+ chdir: True # change to output folder
+ job_logging:
+ formatters:
+ simple:
+ format: '[%(levelname)s] - %(message)s'
+
+repo_id: bezzam/dummy-dataset
+seed: 0
+test_size: 0.15
+hf_token:
+
+data_dir:
+ audio:
+ dir: dataset/data_audio
+ type: wav
+ image:
+ dir: dataset/data_images
+ type: png
+ text:
+ dir: dataset/data_text
+ type: txt
+ label:
+ file: dataset/data_labels.csv
+ label: True
+
+stratify_by_column: label

examples/create_huggingface_dataset.py

@@ -0,0 +1,232 @@
+"""
+We will create a dataset with images, audios, and text data
+so that you can see how various data types can be pushed to
+Hugging Face!
+
+The default configuration is in `examples/configs/hf_dataset.yaml`:
+
+```bash
+# install
+pip install datasets huggingface_hub soundfile
+
+# make a WRITE token on HuggingFace: https://huggingface.co/settings/tokens
+
+# run
+python examples/create_huggingface_dataset.py \
+hf_token=... \
+```
+"""
+
+import hydra
+from hydra.utils import to_absolute_path
+import os
+import time
+import glob
+import numpy as np
+import soundfile as sf
+from PIL import Image as PILImage
+from datasets import Dataset, Image, Audio, ClassLabel
+from omegaconf import open_dict
+from huggingface_hub import upload_file
+import re
+import pandas as pd
+
+
+# -- helper functions
+def convert(text):
+ return int(text) if text.isdigit() else text.lower()
+
+
+def alphanum_key(key):
+ return [convert(c) for c in re.split("([0-9]+)", key)]
+
+
+def natural_sort(arr):
+ return sorted(arr, key=alphanum_key)
+
+
+@hydra.main(version_base=None, config_path="configs", config_name="hf_dataset")
+def main(config):
+
+ start_time = time.time()
+
+ # extract and check parameters
+ repo_id = config.repo_id
+ hf_token = config.hf_token
+ test_size = config.test_size
+
+ assert repo_id is not None, "Please provide a Hugging Face repo_id."
+ assert hf_token is not None, "Please provide a Hugging Face token."
+
+ # to absolute path, as needed by Hugging Face upload
+ for data in config.data_dir:
+ if "dir" in config.data_dir[data]:
+ config.data_dir[data]["dir"] = to_absolute_path(config.data_dir[data]["dir"])
+ elif "file" in config.data_dir[data]:
+ config.data_dir[data]["file"] = to_absolute_path(config.data_dir[data]["file"])
+
+ # Step 1: Check data (create dummy data if not present)
+ n_files = 100 # number of dummy files to create
+ for data in config.data_dir:
+
+ # for directory of data
+ if "dir" in config.data_dir[data]:
+ input_dir = config.data_dir[data]["dir"]
+ data_type = config.data_dir[data]["type"]
+
+ if not os.path.exists(input_dir):
+ # create dummy data
+ print(f"-- Creating {n_files} dummy {data_type} files in {input_dir}")
+ os.makedirs(input_dir, exist_ok=True)
+ for i in range(n_files):
+ if data_type == "png":
+ dim = np.random.randint(100, 200)
+ img = np.random.randint(0, 255, (dim, dim, 3), dtype=np.uint8)
+ img_path = os.path.join(input_dir, f"{i}.png")
+ PILImage.fromarray(img).save(img_path)
+ elif data_type == "wav":
+ duration = np.random.randint(1, 4)
+ sample_rate = 16000
+ audio = np.random.randn(duration * sample_rate)
+ audio_path = os.path.join(input_dir, f"{i}.wav")
+ sf.write(audio_path, audio, samplerate=sample_rate)
+ elif data_type == "txt":
+ text = f"Hello, this is file {i}"
+ text_path = os.path.join(input_dir, f"{i}.txt")
+ with open(text_path, "w") as f:
+ f.write(text)
+
+ # check number of files
+ files = glob.glob(os.path.join(input_dir, "*." + data_type))
+ n_files = len(files)
+ print(f"Found {n_files} {data_type} files in {input_dir}")
+
+ # for CSV file where each line is a data point
+ elif "file" in config.data_dir[data]:
+ input_file = config.data_dir[data]["file"]
+
+ if not os.path.exists(input_file):
+ # create dummy labels
+ labels = ["good", "ok", "bad"]
+ file_labels = np.random.choice(labels, n_files)
+ with open(input_file, "w") as f:
+ for i in range(n_files):
+ f.write(f"{i},{file_labels[i]}\n")
+ print(f"-- Created dummy labels file at {input_file}")
+
+ # check number of unique labels (open with Pandas)
+ df = pd.read_csv(input_file, header=None)
+ n_files = len(df)
+ labels = df[1].unique()
+ n_labels = len(df[1].unique())
+ print(f"Found {n_files} lines with {n_labels} unique labels ({labels}) in {input_file}")
+
+ else:
+ raise ValueError("Please provide either `dir` or `file` in data_dir")
+
+ # -- only keep common files across all datasets
+ bn = [os.path.basename(f).split(".")[0] for f in files]
+ for data in config.data_dir:
+ if "dir" in config.data_dir[data]:
+ input_dir = config.data_dir[data]["dir"]
+ data_type = config.data_dir[data]["type"]
+ files = glob.glob(os.path.join(input_dir, "*." + data_type))
+ bn_data = [os.path.basename(f).split(".")[0] for f in files]
+ common_files = list(set(bn).intersection(bn_data))
+ common_files = natural_sort(common_files)
+ print(f"Number of common files: {len(common_files)}")
+
+ # -- add common files into dictionary
+ for data in config.data_dir:
+ if "dir" in config.data_dir[data]:
+ with open_dict(config):
+ config.data_dir[data]["data"] = common_files
+ if "file" in config.data_dir[data]:
+ # take row according to common_files
+ df = pd.read_csv(config.data_dir[data]["file"], header=None)
+ # -- make first column string
+ df[0] = df[0].astype(str)
+ df = df[df[0].isin(common_files)]
+ with open_dict(config):
+ config.data_dir[data]["data"] = df[1].tolist()
+
+ # Step 2: Create train and test data
+ dataset_dict = {}
+
+ # -- create dictionary of content
+ for data in config.data_dir:
+ if "dir" in config.data_dir[data]:
+ files = config.data_dir[data]["data"]
+ data_type = config.data_dir[data]["type"]
+ data_files = [
+ os.path.join(config.data_dir[data]["dir"], f"{f}.{data_type}") for f in files
+ ]
+
+ if data_type in ["txt"]:
+ # open file content for text files
+ data_files = [open(f).read() for f in data_files]
+ dataset_dict[data] = data_files
+ elif "file" in config.data_dir[data]:
+ dataset_dict[data] = config.data_dir[data]["data"]
+
+ # -- create dataset
+ dataset = Dataset.from_dict(dataset_dict)
+ for data in config.data_dir:
+ if "dir" in config.data_dir[data]:
+ if config.data_dir[data]["type"] in ["png", "jpg", "jpeg", "tiff"]:
+ dataset = dataset.cast_column(data, Image())
+ elif config.data_dir[data]["type"] in ["wav", "mp3", "flac", "ogg"]:
+ dataset = dataset.cast_column(data, Audio())
+ elif "file" in config.data_dir[data]:
+ if config.data_dir[data]["label"]:
+ labels = list(set(config.data_dir[data]["data"]))
+ dataset = dataset.cast_column(data, ClassLabel(names=labels))
+
+ # -- split into train and test
+ dataset = dataset.train_test_split(
+ test_size=test_size,
+ seed=config.seed,
+ shuffle=True,
+ stratify_by_column=config.stratify_by_column, # shuffle must be True
+ )
+ print(dataset)
+
+ """
+ DatasetDict({
+ train: Dataset({
+ features: ['audio', 'images', 'text', 'labels'],
+ num_rows: 85
+ })
+ test: Dataset({
+ features: ['audio', 'images', 'text', 'labels'],
+ num_rows: 15
+ })
+ })
+ """
+
+ # Step 3: Push to Hugging Face
+ dataset.push_to_hub(repo_id, token=hf_token)
+
+ # -- push individual files
+ for data in config.data_dir:
+ if "dir" in config.data_dir[data]:
+ # push first file
+ local_fp = os.path.join(
+ config.data_dir[data]["dir"],
+ config.data_dir[data]["data"][0] + "." + config.data_dir[data]["type"],
+ )
+ remote_fn = "example." + config.data_dir[data]["type"]
+ upload_file(
+ path_or_fileobj=local_fp,
+ path_in_repo=remote_fn,
+ repo_id=repo_id,
+ repo_type="dataset",
+ token=hf_token,
+ )
+
+ # total time in minutes
+ print(f"Total time: {(time.time() - start_time) / 60} minutes")
+
+
+if __name__ == "__main__":
+ main()

examples/use_huggingface_dataset.py

@@ -0,0 +1,50 @@
+"""
+In this script, we use the Hugging Face dataset made
+from the script examples/create_huggingface_dataset.py
+
+The dataset is available at:
+https://huggingface.co/datasets/bezzam/dummy-dataset
+
+```bash
+# install
+pip install datasets librosa soundfile
+
+# run
+python examples/use_huggingface_dataset.py
+```
+
+During the first run, the dataset will be downloaded and cached.
+Subsequent runs will use the cached dataset.
+
+"""
+
+from datasets import load_dataset
+import numpy as np
+
+
+# load train and test splits
+ds_train = load_dataset("bezzam/dummy-dataset", split="train")
+ds_test = load_dataset("bezzam/dummy-dataset", split="test")
+print(f"Number of training samples: {len(ds_train)}")
+print(f"Number of test samples: {len(ds_test)}")
+
+# load first example
+print("\n---- First example:")
+example = ds_train[0]
+
+# -- audio duration
+duration = len(example["audio"]["array"]) / example["audio"]["sampling_rate"]
+print(f"Duration of audio: {duration:.2f} seconds")
+
+# -- image size
+image = np.array(example["image"])
+print(f"Size of image: {image.shape}")
+
+# -- text
+text = example["text"]
+print(f"Text: {text}")
+
+# -- label
+label = example["label"]
+label_str = ds_train.features["label"].int2str(label)
+print(f"Label: {label_str}")

.gitignore

@@ -1,5 +1,6 @@
 project_env/
 .DS_Store
+dataset/*
 
 # from hydra
 outputs/ 

README.rst

@@ -61,7 +61,8 @@ choose the features that you like. This flexibility is one of the best
 * Code formatting.
 * Unit tests and continuous integration.
 * Packaging and distribution.
-* Remove development.
+* Remote development.
+* Creating and sharing datasets with Hugging Face.
 
 The accompanying 
 `slides <https://docs.google.com/presentation/d/1D1_JywMl2rjaeuVzpykPBOJsDIuwQKGOJB4EFZjej2s/edit#slide=id.g2eaa4b61f15_0_1346>`__ 

examples/configs/hf_dataset.yaml

@@ -0,0 +1,29 @@
+# python examples/create_huggingface_dataset.py hf_token=YOUR_TOKEN
+hydra:
+ job:
+ chdir: True # change to output folder
+ job_logging:
+ formatters:
+ simple:
+ format: '[%(levelname)s] - %(message)s'
+
+repo_id: bezzam/dummy-dataset
+seed: 0
+test_size: 0.15
+hf_token:
+
+data_dir:
+ audio:
+ dir: dataset/data_audio
+ type: wav
+ image:
+ dir: dataset/data_images
+ type: png
+ text:
+ dir: dataset/data_text
+ type: txt
+ label:
+ file: dataset/data_labels.csv
+ label: True
+
+stratify_by_column: label

examples/create_huggingface_dataset.py

@@ -0,0 +1,232 @@
+"""
+We will create a dataset with images, audios, and text data
+so that you can see how various data types can be pushed to
+Hugging Face!
+
+The default configuration is in `examples/configs/hf_dataset.yaml`:
+
+```bash
+# install
+pip install datasets huggingface_hub soundfile
+
+# make a WRITE token on HuggingFace: https://huggingface.co/settings/tokens
+
+# run
+python examples/create_huggingface_dataset.py \
+hf_token=... \
+```
+"""
+
+import hydra
+from hydra.utils import to_absolute_path
+import os
+import time
+import glob
+import numpy as np
+import soundfile as sf
+from PIL import Image as PILImage
+from datasets import Dataset, Image, Audio, ClassLabel
+from omegaconf import open_dict
+from huggingface_hub import upload_file
+import re
+import pandas as pd
+
+
+# -- helper functions
+def convert(text):
+ return int(text) if text.isdigit() else text.lower()
+
+
+def alphanum_key(key):
+ return [convert(c) for c in re.split("([0-9]+)", key)]
+
+
+def natural_sort(arr):
+ return sorted(arr, key=alphanum_key)
+
+
+@hydra.main(version_base=None, config_path="configs", config_name="hf_dataset")
+def main(config):
+
+ start_time = time.time()
+
+ # extract and check parameters
+ repo_id = config.repo_id
+ hf_token = config.hf_token
+ test_size = config.test_size
+
+ assert repo_id is not None, "Please provide a Hugging Face repo_id."
+ assert hf_token is not None, "Please provide a Hugging Face token."
+
+ # to absolute path, as needed by Hugging Face upload
+ for data in config.data_dir:
+ if "dir" in config.data_dir[data]:
+ config.data_dir[data]["dir"] = to_absolute_path(config.data_dir[data]["dir"])
+ elif "file" in config.data_dir[data]:
+ config.data_dir[data]["file"] = to_absolute_path(config.data_dir[data]["file"])
+
+ # Step 1: Check data (create dummy data if not present)
+ n_files = 100 # number of dummy files to create
+ for data in config.data_dir:
+
+ # for directory of data
+ if "dir" in config.data_dir[data]:
+ input_dir = config.data_dir[data]["dir"]
+ data_type = config.data_dir[data]["type"]
+
+ if not os.path.exists(input_dir):
+ # create dummy data
+ print(f"-- Creating {n_files} dummy {data_type} files in {input_dir}")
+ os.makedirs(input_dir, exist_ok=True)
+ for i in range(n_files):
+ if data_type == "png":
+ dim = np.random.randint(100, 200)
+ img = np.random.randint(0, 255, (dim, dim, 3), dtype=np.uint8)
+ img_path = os.path.join(input_dir, f"{i}.png")
+ PILImage.fromarray(img).save(img_path)
+ elif data_type == "wav":
+ duration = np.random.randint(1, 4)
+ sample_rate = 16000
+ audio = np.random.randn(duration * sample_rate)
+ audio_path = os.path.join(input_dir, f"{i}.wav")
+ sf.write(audio_path, audio, samplerate=sample_rate)
+ elif data_type == "txt":
+ text = f"Hello, this is file {i}"
+ text_path = os.path.join(input_dir, f"{i}.txt")
+ with open(text_path, "w") as f:
+ f.write(text)
+
+ # check number of files
+ files = glob.glob(os.path.join(input_dir, "*." + data_type))
+ n_files = len(files)
+ print(f"Found {n_files} {data_type} files in {input_dir}")
+
+ # for CSV file where each line is a data point
+ elif "file" in config.data_dir[data]:
+ input_file = config.data_dir[data]["file"]
+
+ if not os.path.exists(input_file):
+ # create dummy labels
+ labels = ["good", "ok", "bad"]
+ file_labels = np.random.choice(labels, n_files)
+ with open(input_file, "w") as f:
+ for i in range(n_files):
+ f.write(f"{i},{file_labels[i]}\n")
+ print(f"-- Created dummy labels file at {input_file}")
+
+ # check number of unique labels (open with Pandas)
+ df = pd.read_csv(input_file, header=None)
+ n_files = len(df)
+ labels = df[1].unique()
+ n_labels = len(df[1].unique())
+ print(f"Found {n_files} lines with {n_labels} unique labels ({labels}) in {input_file}")
+
+ else:
+ raise ValueError("Please provide either `dir` or `file` in data_dir")
+
+ # -- only keep common files across all datasets
+ bn = [os.path.basename(f).split(".")[0] for f in files]
+ for data in config.data_dir:
+ if "dir" in config.data_dir[data]:
+ input_dir = config.data_dir[data]["dir"]
+ data_type = config.data_dir[data]["type"]
+ files = glob.glob(os.path.join(input_dir, "*." + data_type))
+ bn_data = [os.path.basename(f).split(".")[0] for f in files]
+ common_files = list(set(bn).intersection(bn_data))
+ common_files = natural_sort(common_files)
+ print(f"Number of common files: {len(common_files)}")
+
+ # -- add common files into dictionary
+ for data in config.data_dir:
+ if "dir" in config.data_dir[data]:
+ with open_dict(config):
+ config.data_dir[data]["data"] = common_files
+ if "file" in config.data_dir[data]:
+ # take row according to common_files
+ df = pd.read_csv(config.data_dir[data]["file"], header=None)
+ # -- make first column string
+ df[0] = df[0].astype(str)
+ df = df[df[0].isin(common_files)]
+ with open_dict(config):
+ config.data_dir[data]["data"] = df[1].tolist()
+
+ # Step 2: Create train and test data
+ dataset_dict = {}
+
+ # -- create dictionary of content
+ for data in config.data_dir:
+ if "dir" in config.data_dir[data]:
+ files = config.data_dir[data]["data"]
+ data_type = config.data_dir[data]["type"]
+ data_files = [
+ os.path.join(config.data_dir[data]["dir"], f"{f}.{data_type}") for f in files
+ ]
+
+ if data_type in ["txt"]:
+ # open file content for text files
+ data_files = [open(f).read() for f in data_files]
+ dataset_dict[data] = data_files
+ elif "file" in config.data_dir[data]:
+ dataset_dict[data] = config.data_dir[data]["data"]
+
+ # -- create dataset
+ dataset = Dataset.from_dict(dataset_dict)
+ for data in config.data_dir:
+ if "dir" in config.data_dir[data]:
+ if config.data_dir[data]["type"] in ["png", "jpg", "jpeg", "tiff"]:
+ dataset = dataset.cast_column(data, Image())
+ elif config.data_dir[data]["type"] in ["wav", "mp3", "flac", "ogg"]:
+ dataset = dataset.cast_column(data, Audio())
+ elif "file" in config.data_dir[data]:
+ if config.data_dir[data]["label"]:
+ labels = list(set(config.data_dir[data]["data"]))
+ dataset = dataset.cast_column(data, ClassLabel(names=labels))
+
+ # -- split into train and test
+ dataset = dataset.train_test_split(
+ test_size=test_size,
+ seed=config.seed,
+ shuffle=True,
+ stratify_by_column=config.stratify_by_column, # shuffle must be True
+ )
+ print(dataset)
+
+ """
+ DatasetDict({
+ train: Dataset({
+ features: ['audio', 'images', 'text', 'labels'],
+ num_rows: 85
+ })
+ test: Dataset({
+ features: ['audio', 'images', 'text', 'labels'],
+ num_rows: 15
+ })
+ })
+ """
+
+ # Step 3: Push to Hugging Face
+ dataset.push_to_hub(repo_id, token=hf_token)
+
+ # -- push individual files
+ for data in config.data_dir:
+ if "dir" in config.data_dir[data]:
+ # push first file
+ local_fp = os.path.join(
+ config.data_dir[data]["dir"],
+ config.data_dir[data]["data"][0] + "." + config.data_dir[data]["type"],
+ )
+ remote_fn = "example." + config.data_dir[data]["type"]
+ upload_file(
+ path_or_fileobj=local_fp,
+ path_in_repo=remote_fn,
+ repo_id=repo_id,
+ repo_type="dataset",
+ token=hf_token,
+ )
+
+ # total time in minutes
+ print(f"Total time: {(time.time() - start_time) / 60} minutes")
+
+
+if __name__ == "__main__":
+ main()

examples/use_huggingface_dataset.py

@@ -0,0 +1,50 @@
+"""
+In this script, we use the Hugging Face dataset made
+from the script examples/create_huggingface_dataset.py
+
+The dataset is available at:
+https://huggingface.co/datasets/bezzam/dummy-dataset
+
+```bash
+# install
+pip install datasets librosa soundfile
+
+# run
+python examples/use_huggingface_dataset.py
+```
+
+During the first run, the dataset will be downloaded and cached.
+Subsequent runs will use the cached dataset.
+
+"""
+
+from datasets import load_dataset
+import numpy as np
+
+
+# load train and test splits
+ds_train = load_dataset("bezzam/dummy-dataset", split="train")
+ds_test = load_dataset("bezzam/dummy-dataset", split="test")
+print(f"Number of training samples: {len(ds_train)}")
+print(f"Number of test samples: {len(ds_test)}")
+
+# load first example
+print("\n---- First example:")
+example = ds_train[0]
+
+# -- audio duration
+duration = len(example["audio"]["array"]) / example["audio"]["sampling_rate"]
+print(f"Duration of audio: {duration:.2f} seconds")
+
+# -- image size
+image = np.array(example["image"])
+print(f"Size of image: {image.shape}")
+
+# -- text
+text = example["text"]
+print(f"Text: {text}")
+
+# -- label
+label = example["label"]
+label_str = ds_train.features["label"].int2str(label)
+print(f"Label: {label_str}")

.gitignore

@@ -1,5 +1,6 @@
 project_env/
 .DS_Store
+dataset/*
 
 # from hydra
 outputs/ 

README.rst

@@ -61,7 +61,8 @@ choose the features that you like. This flexibility is one of the best
 * Code formatting.
 * Unit tests and continuous integration.
 * Packaging and distribution.
-* Remove development.
+* Remote development.
+* Creating and sharing datasets with Hugging Face.
 
 The accompanying 
 `slides <https://docs.google.com/presentation/d/1D1_JywMl2rjaeuVzpykPBOJsDIuwQKGOJB4EFZjej2s/edit#slide=id.g2eaa4b61f15_0_1346>`__ 

examples/configs/hf_dataset.yaml

@@ -0,0 +1,29 @@
+# python examples/create_huggingface_dataset.py hf_token=YOUR_TOKEN
+hydra:
+ job:
+ chdir: True # change to output folder
+ job_logging:
+ formatters:
+ simple:
+ format: '[%(levelname)s] - %(message)s'
+
+repo_id: bezzam/dummy-dataset
+seed: 0
+test_size: 0.15
+hf_token:
+
+data_dir:
+ audio:
+ dir: dataset/data_audio
+ type: wav
+ image:
+ dir: dataset/data_images
+ type: png
+ text:
+ dir: dataset/data_text
+ type: txt
+ label:
+ file: dataset/data_labels.csv
+ label: True
+
+stratify_by_column: label

examples/create_huggingface_dataset.py

@@ -0,0 +1,232 @@
+"""
+We will create a dataset with images, audios, and text data
+so that you can see how various data types can be pushed to
+Hugging Face!
+
+The default configuration is in `examples/configs/hf_dataset.yaml`:
+
+```bash
+# install
+pip install datasets huggingface_hub soundfile
+
+# make a WRITE token on HuggingFace: https://huggingface.co/settings/tokens
+
+# run
+python examples/create_huggingface_dataset.py \
+hf_token=... \
+```
+"""
+
+import hydra
+from hydra.utils import to_absolute_path
+import os
+import time
+import glob
+import numpy as np
+import soundfile as sf
+from PIL import Image as PILImage
+from datasets import Dataset, Image, Audio, ClassLabel
+from omegaconf import open_dict
+from huggingface_hub import upload_file
+import re
+import pandas as pd
+
+
+# -- helper functions
+def convert(text):
+ return int(text) if text.isdigit() else text.lower()
+
+
+def alphanum_key(key):
+ return [convert(c) for c in re.split("([0-9]+)", key)]
+
+
+def natural_sort(arr):
+ return sorted(arr, key=alphanum_key)
+
+
+@hydra.main(version_base=None, config_path="configs", config_name="hf_dataset")
+def main(config):
+
+ start_time = time.time()
+
+ # extract and check parameters
+ repo_id = config.repo_id
+ hf_token = config.hf_token
+ test_size = config.test_size
+
+ assert repo_id is not None, "Please provide a Hugging Face repo_id."
+ assert hf_token is not None, "Please provide a Hugging Face token."
+
+ # to absolute path, as needed by Hugging Face upload
+ for data in config.data_dir:
+ if "dir" in config.data_dir[data]:
+ config.data_dir[data]["dir"] = to_absolute_path(config.data_dir[data]["dir"])
+ elif "file" in config.data_dir[data]:
+ config.data_dir[data]["file"] = to_absolute_path(config.data_dir[data]["file"])
+
+ # Step 1: Check data (create dummy data if not present)
+ n_files = 100 # number of dummy files to create
+ for data in config.data_dir:
+
+ # for directory of data
+ if "dir" in config.data_dir[data]:
+ input_dir = config.data_dir[data]["dir"]
+ data_type = config.data_dir[data]["type"]
+
+ if not os.path.exists(input_dir):
+ # create dummy data
+ print(f"-- Creating {n_files} dummy {data_type} files in {input_dir}")
+ os.makedirs(input_dir, exist_ok=True)
+ for i in range(n_files):
+ if data_type == "png":
+ dim = np.random.randint(100, 200)
+ img = np.random.randint(0, 255, (dim, dim, 3), dtype=np.uint8)
+ img_path = os.path.join(input_dir, f"{i}.png")
+ PILImage.fromarray(img).save(img_path)
+ elif data_type == "wav":
+ duration = np.random.randint(1, 4)
+ sample_rate = 16000
+ audio = np.random.randn(duration * sample_rate)
+ audio_path = os.path.join(input_dir, f"{i}.wav")
+ sf.write(audio_path, audio, samplerate=sample_rate)
+ elif data_type == "txt":
+ text = f"Hello, this is file {i}"
+ text_path = os.path.join(input_dir, f"{i}.txt")
+ with open(text_path, "w") as f:
+ f.write(text)
+
+ # check number of files
+ files = glob.glob(os.path.join(input_dir, "*." + data_type))
+ n_files = len(files)
+ print(f"Found {n_files} {data_type} files in {input_dir}")
+
+ # for CSV file where each line is a data point
+ elif "file" in config.data_dir[data]:
+ input_file = config.data_dir[data]["file"]
+
+ if not os.path.exists(input_file):
+ # create dummy labels
+ labels = ["good", "ok", "bad"]
+ file_labels = np.random.choice(labels, n_files)
+ with open(input_file, "w") as f:
+ for i in range(n_files):
+ f.write(f"{i},{file_labels[i]}\n")
+ print(f"-- Created dummy labels file at {input_file}")
+
+ # check number of unique labels (open with Pandas)
+ df = pd.read_csv(input_file, header=None)
+ n_files = len(df)
+ labels = df[1].unique()
+ n_labels = len(df[1].unique())
+ print(f"Found {n_files} lines with {n_labels} unique labels ({labels}) in {input_file}")
+
+ else:
+ raise ValueError("Please provide either `dir` or `file` in data_dir")
+
+ # -- only keep common files across all datasets
+ bn = [os.path.basename(f).split(".")[0] for f in files]
+ for data in config.data_dir:
+ if "dir" in config.data_dir[data]:
+ input_dir = config.data_dir[data]["dir"]
+ data_type = config.data_dir[data]["type"]
+ files = glob.glob(os.path.join(input_dir, "*." + data_type))
+ bn_data = [os.path.basename(f).split(".")[0] for f in files]
+ common_files = list(set(bn).intersection(bn_data))
+ common_files = natural_sort(common_files)
+ print(f"Number of common files: {len(common_files)}")
+
+ # -- add common files into dictionary
+ for data in config.data_dir:
+ if "dir" in config.data_dir[data]:
+ with open_dict(config):
+ config.data_dir[data]["data"] = common_files
+ if "file" in config.data_dir[data]:
+ # take row according to common_files
+ df = pd.read_csv(config.data_dir[data]["file"], header=None)
+ # -- make first column string
+ df[0] = df[0].astype(str)
+ df = df[df[0].isin(common_files)]
+ with open_dict(config):
+ config.data_dir[data]["data"] = df[1].tolist()
+
+ # Step 2: Create train and test data
+ dataset_dict = {}
+
+ # -- create dictionary of content
+ for data in config.data_dir:
+ if "dir" in config.data_dir[data]:
+ files = config.data_dir[data]["data"]
+ data_type = config.data_dir[data]["type"]
+ data_files = [
+ os.path.join(config.data_dir[data]["dir"], f"{f}.{data_type}") for f in files
+ ]
+
+ if data_type in ["txt"]:
+ # open file content for text files
+ data_files = [open(f).read() for f in data_files]
+ dataset_dict[data] = data_files
+ elif "file" in config.data_dir[data]:
+ dataset_dict[data] = config.data_dir[data]["data"]
+
+ # -- create dataset
+ dataset = Dataset.from_dict(dataset_dict)
+ for data in config.data_dir:
+ if "dir" in config.data_dir[data]:
+ if config.data_dir[data]["type"] in ["png", "jpg", "jpeg", "tiff"]:
+ dataset = dataset.cast_column(data, Image())
+ elif config.data_dir[data]["type"] in ["wav", "mp3", "flac", "ogg"]:
+ dataset = dataset.cast_column(data, Audio())
+ elif "file" in config.data_dir[data]:
+ if config.data_dir[data]["label"]:
+ labels = list(set(config.data_dir[data]["data"]))
+ dataset = dataset.cast_column(data, ClassLabel(names=labels))
+
+ # -- split into train and test
+ dataset = dataset.train_test_split(
+ test_size=test_size,
+ seed=config.seed,
+ shuffle=True,
+ stratify_by_column=config.stratify_by_column, # shuffle must be True
+ )
+ print(dataset)
+
+ """
+ DatasetDict({
+ train: Dataset({
+ features: ['audio', 'images', 'text', 'labels'],
+ num_rows: 85
+ })
+ test: Dataset({
+ features: ['audio', 'images', 'text', 'labels'],
+ num_rows: 15
+ })
+ })
+ """
+
+ # Step 3: Push to Hugging Face
+ dataset.push_to_hub(repo_id, token=hf_token)
+
+ # -- push individual files
+ for data in config.data_dir:
+ if "dir" in config.data_dir[data]:
+ # push first file
+ local_fp = os.path.join(
+ config.data_dir[data]["dir"],
+ config.data_dir[data]["data"][0] + "." + config.data_dir[data]["type"],
+ )
+ remote_fn = "example." + config.data_dir[data]["type"]
+ upload_file(
+ path_or_fileobj=local_fp,
+ path_in_repo=remote_fn,
+ repo_id=repo_id,
+ repo_type="dataset",
+ token=hf_token,
+ )
+
+ # total time in minutes
+ print(f"Total time: {(time.time() - start_time) / 60} minutes")
+
+
+if __name__ == "__main__":
+ main()

examples/use_huggingface_dataset.py

@@ -0,0 +1,50 @@
+"""
+In this script, we use the Hugging Face dataset made
+from the script examples/create_huggingface_dataset.py
+
+The dataset is available at:
+https://huggingface.co/datasets/bezzam/dummy-dataset
+
+```bash
+# install
+pip install datasets librosa soundfile
+
+# run
+python examples/use_huggingface_dataset.py
+```
+
+During the first run, the dataset will be downloaded and cached.
+Subsequent runs will use the cached dataset.
+
+"""
+
+from datasets import load_dataset
+import numpy as np
+
+
+# load train and test splits
+ds_train = load_dataset("bezzam/dummy-dataset", split="train")
+ds_test = load_dataset("bezzam/dummy-dataset", split="test")
+print(f"Number of training samples: {len(ds_train)}")
+print(f"Number of test samples: {len(ds_test)}")
+
+# load first example
+print("\n---- First example:")
+example = ds_train[0]
+
+# -- audio duration
+duration = len(example["audio"]["array"]) / example["audio"]["sampling_rate"]
+print(f"Duration of audio: {duration:.2f} seconds")
+
+# -- image size
+image = np.array(example["image"])
+print(f"Size of image: {image.shape}")
+
+# -- text
+text = example["text"]
+print(f"Text: {text}")
+
+# -- label
+label = example["label"]
+label_str = ds_train.features["label"].int2str(label)
+print(f"Label: {label_str}")