Skip to content

download_unicode_data_files.py: Select latest or a specific version #4469

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Stephan T. Lavavej (StephanTLavavej) merged 2 commits into from
Mar 16, 2024
Merged

download_unicode_data_files.py: Select latest or a specific version #4469

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7d3cad8
Update script to download `latest` or a specific version.
StephanTLavavej Mar 11, 2024
1dd22be
Print HTTPError nicely.
StephanTLavavej Mar 11, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 35 additions & 11 deletions tools/unicode_properties_parse/download_unicode_data_files.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,22 +1,46 @@
# Copyright (c) Microsoft Corporation.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception

from pathlib import PurePosixPath
import sys
from urllib.error import HTTPError
from urllib.request import urlretrieve


Unicode_data_files = {
"DerivedCoreProperties.txt": "https://www.unicode.org/Public/UCD/latest/ucd/DerivedCoreProperties.txt",
"DerivedGeneralCategory.txt": "https://www.unicode.org/Public/UCD/latest/ucd/extracted/DerivedGeneralCategory.txt",
"EastAsianWidth.txt": "https://www.unicode.org/Public/UCD/latest/ucd/EastAsianWidth.txt",
"GraphemeBreakProperty.txt": "https://www.unicode.org/Public/UCD/latest/ucd/auxiliary/GraphemeBreakProperty.txt",
"GraphemeBreakTest.txt": "https://www.unicode.org/Public/UCD/latest/ucd/auxiliary/GraphemeBreakTest.txt",
"emoji-data.txt": "https://www.unicode.org/Public/UCD/latest/ucd/emoji/emoji-data.txt",
}
def get_base_url():
if len(sys.argv) != 2:
sys.exit(f"Usage: python {sys.argv[0]} [latest|<VERSION LIKE 15.0.0>]")

version = sys.argv[1]

if version == "latest":
return "https://unicode.org/Public/UCD/latest/"

return f"https://unicode.org/Public/{version}/"


Unicode_data_files = [
"ucd/DerivedCoreProperties.txt",
"ucd/extracted/DerivedGeneralCategory.txt",
"ucd/EastAsianWidth.txt",
"ucd/auxiliary/GraphemeBreakProperty.txt",
"ucd/auxiliary/GraphemeBreakTest.txt",
"ucd/emoji/emoji-data.txt",
]


def download_unicode_data_files():
for filename, url in Unicode_data_files.items():
print(f"downloading {filename} from {url}")
urlretrieve(url, filename)
base_url = get_base_url()
print(f" Base URL: {base_url}")

for data_file in Unicode_data_files:
url = base_url + data_file
filename = PurePosixPath(data_file).name
print(f"Downloading: {url}")
try:
urlretrieve(url, filename)
except HTTPError as http_error:
sys.exit(f"{http_error}")


if __name__ == "__main__":
Expand Down