From e4e742079070c8226e33ab980bf90813784b28fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=B5=20V=C4=83n=20Ngh=C4=A9a?= Date: Mon, 3 May 2021 00:50:42 +0200 Subject: [PATCH 1/4] add initial tests for s3 --- tests/test_filesystem.py | 357 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 357 insertions(+) create mode 100644 tests/test_filesystem.py diff --git a/tests/test_filesystem.py b/tests/test_filesystem.py new file mode 100644 index 000000000..8b5809bf7 --- /dev/null +++ b/tests/test_filesystem.py @@ -0,0 +1,357 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +# ============================================================================== + +import os +import posixpath +import time +from urllib.parse import urlparse + +import boto3 +import pytest +import tensorflow as tf +import tensorflow_io as tfio # pylint: disable=unused-import + +# pytestmark = pytest.mark.skipif( +# sys.platform in ("win32", "darwin"), +# reason="TODO emulator not setup properly on macOS/Windows yet", +# ) + +ROOT_PREFIX = f"tf-io-root-{int(time.time())}/" +S3_URI = "s3" + + +def mock_patchs(monkeypatch, patchs): + if isinstance(patchs, dict): + for key, value in patchs.items(): + if value is None: + monkeypatch.delenv(key, raising=False) + else: + monkeypatch.setenv(key, value) + elif callable(patchs): + patchs(monkeypatch) + else: + pass + + +@pytest.fixture(autouse=True) +def reload_filesystem(): + # We need to find a way to reload `tensorflow` or filesystems since + # the envs are read in the first time the filesystems are called. + pass + + +@pytest.fixture(scope="module") +def s3_fs(): + monkeypatch = pytest.MonkeyPatch() + bucket_name = os.environ.get("S3_TEST_BUCKET") + client = None + + # This means we are running against emulator. + if bucket_name is None: + endpoint_url = "http://localhost:4566" + monkeypatch.setenv("AWS_REGION", "us-east-1") + monkeypatch.setenv("AWS_ACCESS_KEY_ID", "TEST") + monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "TEST") + monkeypatch.setenv("S3_ENDPOINT", endpoint_url) + + bucket_name = f"tf-io-bucket-s3-{int(time.time())}" + client = boto3.client("s3", endpoint_url=endpoint_url) + client.create_bucket(Bucket=bucket_name) + else: + # TODO(vnvo2409): Implement for testing against production scenario + pass + + client.put_object(Bucket=bucket_name, Key=ROOT_PREFIX, Body="") + + def parse(path): + res = urlparse(path, scheme=S3_URI, allow_fragments=False) + return res.netloc, res.path[1:] + + def path_to(*args): + return f"{S3_URI}://{bucket_name}/{posixpath.join(ROOT_PREFIX, *args)}" + + def read(path): + bucket_name, key_name = parse(path) + response = client.get_object(Bucket=bucket_name, Key=key_name) + return response["Body"].read() + + def write(path, body): + bucket_name, key_name = parse(path) + client.put_object(Bucket=bucket_name, Key=key_name, Body=body) + + def mkdirs(path): + if path[-1] != "/": + path += "/" + write(path, b"") + + yield path_to, read, write, mkdirs, posixpath.join + monkeypatch.undo() + + +@pytest.fixture +def fs(request, s3_fs): + if request.param == S3_URI: + return s3_fs + + +@pytest.mark.parametrize("fs, patchs", [(S3_URI, None)], indirect=["fs"]) +def test_init(fs, patchs, monkeypatch): + path_to, _, _, _, _ = fs + mock_patchs(monkeypatch, patchs) + assert tf.io.gfile.exists(path_to("")) is True + + +@pytest.mark.parametrize("fs, patchs", [(S3_URI, None)], indirect=["fs"]) +def test_io_read_file(fs, patchs, monkeypatch): + path_to, _, write, _, _ = fs + mock_patchs(monkeypatch, patchs) + + fname = path_to("test_io_read_file") + body = b"abcdefghijklmn" + write(fname, body) + + assert tf.io.read_file(fname) == body + + +@pytest.mark.parametrize("fs, patchs", [(S3_URI, None)], indirect=["fs"]) +def test_io_write_file(fs, patchs, monkeypatch): + path_to, read, _, _, _ = fs + mock_patchs(monkeypatch, patchs) + + fname = path_to("test_io_write_file") + assert tf.io.gfile.exists(fname) is False + + body = b"abcdefghijklmn" + tf.io.write_file(fname, body) + + assert read(fname) == body + + +@pytest.mark.parametrize( + "fs, patchs", + [ + ( + S3_URI, + # `use_multi_part_download` does not work with `seakable`. + lambda monkeypatch: monkeypatch.setattr( + tf.io.gfile.GFile, "seekable", lambda _: False + ), + ) + ], + indirect=["fs"], +) +def test_gfile_GFile_readable(fs, patchs, monkeypatch): + path_to, _, write, _, _ = fs + mock_patchs(monkeypatch, patchs) + + fname = path_to("test_gfile_GFile_readable") + + num_lines = 10 + base_body = b"abcdefghijklmn\n" + body = base_body * num_lines + write(fname, body) + + # Simple + with tf.io.gfile.GFile(fname, "rb") as f: + file_read = f.read() + assert file_read == body + + # Notfound + with pytest.raises(tf.errors.NotFoundError): + fname_not_found = fname + "_not_found" + with tf.io.gfile.GFile(fname_not_found, "rb") as f: + _ = f.read() + + # Read length + with tf.io.gfile.GFile(fname, "rb") as f: + read_length = 10 + file_read = f.read(read_length) + assert file_read == body[:read_length] + + file_read = f.read() + assert file_read == body[read_length:] + + # Readline + with tf.io.gfile.GFile(fname, "rb") as f: + line_count = 0 + while True: + line = f.readline() + if not line: + break + line_count += 1 + assert line == base_body + assert line_count == num_lines + + # Readlines + with tf.io.gfile.GFile(fname, "rb") as f: + lines = f.readlines() + assert lines == [base_body] * num_lines + + # Seek/Tell + with tf.io.gfile.GFile(fname, "rb") as f: + assert f.size() == len(body) + if f.seekable(): + seek_size = 15 + read_length = 10 + f.seek(seek_size) + file_read = f.read(read_length) + assert f.tell() == seek_size + read_length + assert file_read == body[seek_size : seek_size + read_length] + + +@pytest.mark.parametrize("fs, patchs", [(S3_URI, None)], indirect=["fs"]) +def test_gfile_GFile_writable(fs, patchs, monkeypatch): + path_to, read, _, _, _ = fs + mock_patchs(monkeypatch, patchs) + + fname = path_to("test_gfile_GFile_writable") + assert tf.io.gfile.exists(fname) is False + + num_lines = 10 + base_body = b"abcdefghijklmn\n" + body = base_body * num_lines + + # Simple + with tf.io.gfile.GFile(fname, "wb") as f: + f.write(body) + f.flush() + assert read(fname) == body + + # Append + with tf.io.gfile.GFile(fname, "ab") as f: + f.write(base_body) + f.flush() + assert read(fname) == body + base_body + + +@pytest.mark.parametrize("fs, patchs", [(S3_URI, None)], indirect=["fs"]) +def test_gfile_isdir(fs, patchs, monkeypatch): + path_to, _, write, mkdirs, join = fs + mock_patchs(monkeypatch, patchs) + + root_path = "test_gfile_isdir" + dname = path_to(root_path) + fname = join(dname, "fname") + + mkdirs(dname) + write(fname, b"123456789") + + assert tf.io.gfile.isdir(dname) is True + assert tf.io.gfile.isdir(fname) is False + + +@pytest.mark.parametrize("fs, patchs", [(S3_URI, None)], indirect=["fs"]) +def test_gfile_listdir(fs, patchs, monkeypatch): + path_to, _, write, mkdirs, join = fs + mock_patchs(monkeypatch, patchs) + + root_path = "test_gfile_listdir" + dname = path_to(root_path) + mkdirs(dname) + + num_childs = 5 + childrens = [None] * num_childs + childrens[0] = join(dname, "subdir") + mkdirs(childrens[0]) + + body = b"123456789" + for i in range(1, num_childs): + childrens[i] = join(dname, f"child_{i}") + write(childrens[i], body) + write(join(childrens[0], f"subchild_{i}"), body) + + entries = tf.io.gfile.listdir(dname) + assert sorted(childrens) == sorted([join(dname, entry) for entry in entries]) + + +@pytest.mark.parametrize("fs, patchs", [(S3_URI, None)], indirect=["fs"]) +def test_gfile_makedirs(fs, patchs, monkeypatch): + path_to, _, write, _, join = fs + mock_patchs(monkeypatch, patchs) + + root_path = "test_gfile_makedirs/" + dname = path_to(root_path) + subdname = join(dname, "subdir_1") + assert tf.io.gfile.exists(dname) is False + assert tf.io.gfile.exists(subdname) is False + + tf.io.gfile.mkdir(subdname) + write(join(subdname, "fname"), b"123456789") + assert tf.io.gfile.isdir(subdname) is True + + +@pytest.mark.parametrize("fs, patchs", [(S3_URI, None)], indirect=["fs"]) +def test_gfile_rmtree(fs, patchs, monkeypatch): + path_to, _, write, mkdirs, join = fs + mock_patchs(monkeypatch, patchs) + + num_entries = 3 + trees = [path_to("test_gfile_rmtree")] * num_entries + mkdirs(trees[0]) + + for i in range(1, num_entries - 1): + trees[i] = join(trees[i - 1], f"subdir_{i}") + mkdirs(trees[i]) + + trees[-1] = join(trees[-2], "fname") + write(trees[-1], b"123456789") + + tf.io.gfile.rmtree(trees[0]) + + assert [tf.io.gfile.exists(entry) for entry in trees] == [False] * num_entries + + +@pytest.mark.parametrize("fs, patchs", [(S3_URI, None)], indirect=["fs"]) +def test_gfile_copy(fs, patchs, monkeypatch): + path_to, read, write, _, _ = fs + mock_patchs(monkeypatch, patchs) + + src = path_to("test_gfile_copy_src") + dst = path_to("test_gfile_copy_dst") + + body = b"123456789" + write(src, body) + + tf.io.gfile.copy(src, dst) + assert read(dst) == body + + new_body = body + body.capitalize() + write(src, new_body) + with pytest.raises(tf.errors.AlreadyExistsError): + tf.io.gfile.copy(src, dst) + assert read(dst) == body + + tf.io.gfile.copy(src, dst, overwrite=True) + assert read(dst) == new_body + + +@pytest.mark.parametrize("fs, patchs", [(S3_URI, None)], indirect=["fs"]) +def test_gfile_glob(fs, patchs, monkeypatch): + path_to, _, write, _, join = fs + mock_patchs(monkeypatch, patchs) + + dname = path_to("test_gfile_glob/") + + num_items = 3 + childs = [None] * 3 + for ext in ["txt", "md"]: + for i in range(num_items): + fname = join(dname, f"{i}.{ext}") + if ext == "txt": + childs[i] = fname + write(fname, b"123456789") + + txt_files = tf.io.gfile.glob(join(dname, "*.txt")) + assert sorted(txt_files) == sorted(childs) From 79cf3ffa946366e02f9a5a23f0081ec2724f0741 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=B5=20V=C4=83n=20Ngh=C4=A9a?= Date: Mon, 3 May 2021 01:26:40 +0200 Subject: [PATCH 2/4] add test for az --- .github/workflows/api.yml | 4 +- .github/workflows/benchmarks.yml | 4 +- .github/workflows/build.wheel.sh | 2 +- tests/test_filesystem.py | 147 +++++++++++++++++++++++-------- 4 files changed, 117 insertions(+), 40 deletions(-) diff --git a/.github/workflows/api.yml b/.github/workflows/api.yml index f95d26640..6f57660f1 100644 --- a/.github/workflows/api.yml +++ b/.github/workflows/api.yml @@ -42,7 +42,7 @@ jobs: echo ${{ matrix.version }} | awk -F: '{print $1}' | xargs python -m pip install -U echo ${{ matrix.version }} | awk -F: '{print $2}' | xargs python -m pip install --no-deps -U if [[ "${{ matrix.version }}" == *"tensorflow-io-nightly"* ]]; then python -m pip install tensorflow-io-plugin-gs-nightly ; fi - python -m pip install pytest-benchmark boto3 google-cloud-storage==1.32.0 + python -m pip install pytest-benchmark boto3 google-cloud-storage==1.32.0 azure-storage-blob==12.7.1 python -m pip freeze python -c 'import tensorflow as tf; print(tf.version.VERSION)' python -c 'import tensorflow_io as tfio; print(tfio.version.VERSION)' @@ -80,7 +80,7 @@ jobs: echo ${{ matrix.version }} | awk -F: '{print $1}' | xargs python -m pip install -U echo ${{ matrix.version }} | awk -F: '{print $2}' | xargs python -m pip install --no-deps -U if [[ "${{ matrix.version }}" == *"tensorflow-io-nightly"* ]]; then python -m pip install tensorflow-io-plugin-gs-nightly ; fi - python -m pip install pytest-benchmark boto3 google-cloud-storage==1.32.0 + python -m pip install pytest-benchmark boto3 google-cloud-storage==1.32.0 azure-storage-blob==12.7.1 python -m pip freeze python -c 'import tensorflow as tf; print(tf.version.VERSION)' python -c 'import tensorflow_io as tfio; print(tfio.version.VERSION)' diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 7add09001..317bfd988 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -33,7 +33,7 @@ jobs: echo ${{ matrix.version }} | awk -F: '{print $1}' | xargs python -m pip install -U echo ${{ matrix.version }} | awk -F: '{print $2}' | xargs python -m pip install --no-deps -U if [[ "${{ matrix.version }}" == *"tensorflow-io-nightly"* ]]; then python -m pip install tensorflow-io-plugin-gs-nightly ; fi - python -m pip install -q scikit-image pytest pytest-benchmark boto3 fastavro avro-python3 scikit-image pandas pyarrow==2.0.0 google-cloud-pubsub==2.1.0 google-cloud-bigquery-storage==1.1.0 google-cloud-bigquery==2.3.1 google-cloud-storage==1.32.0 + python -m pip install -q scikit-image pytest pytest-benchmark boto3 fastavro avro-python3 scikit-image pandas pyarrow==2.0.0 google-cloud-pubsub==2.1.0 google-cloud-bigquery-storage==1.1.0 google-cloud-bigquery==2.3.1 google-cloud-storage==1.32.0 azure-storage-blob==12.7.1 python -m pip freeze python -c 'import tensorflow as tf; print(tf.version.VERSION)' python -c 'import tensorflow_io as tfio; print(tfio.version.VERSION)' @@ -68,7 +68,7 @@ jobs: echo ${{ matrix.version }} | awk -F: '{print $1}' | xargs python -m pip install -U echo ${{ matrix.version }} | awk -F: '{print $2}' | xargs python -m pip install --no-deps -U if [[ "${{ matrix.version }}" == *"tensorflow-io-nightly"* ]]; then python -m pip install tensorflow-io-plugin-gs-nightly ; fi - python -m pip install -q scikit-image pytest pytest-benchmark boto3 fastavro avro-python3 scikit-image pandas pyarrow==2.0.0 google-cloud-pubsub==2.1.0 google-cloud-bigquery-storage==1.1.0 google-cloud-bigquery==2.3.1 google-cloud-storage==1.32.0 + python -m pip install -q scikit-image pytest pytest-benchmark boto3 fastavro avro-python3 scikit-image pandas pyarrow==2.0.0 google-cloud-pubsub==2.1.0 google-cloud-bigquery-storage==1.1.0 google-cloud-bigquery==2.3.1 google-cloud-storage==1.32.0 azure-storage-blob==12.7.1 python -m pip freeze python -c 'import tensorflow as tf; print(tf.version.VERSION)' python -c 'import tensorflow_io as tfio; print(tfio.version.VERSION)' diff --git a/.github/workflows/build.wheel.sh b/.github/workflows/build.wheel.sh index 8a2b9951d..dd14b3db5 100755 --- a/.github/workflows/build.wheel.sh +++ b/.github/workflows/build.wheel.sh @@ -8,7 +8,7 @@ run_test() { CPYTHON_VERSION=$($entry -c 'import sys; print(str(sys.version_info[0])+str(sys.version_info[1]))') (cd wheelhouse && $entry -m pip install tensorflow_io_plugin_gs-*-cp${CPYTHON_VERSION}-*.whl) (cd wheelhouse && $entry -m pip install tensorflow_io-*-cp${CPYTHON_VERSION}-*.whl) - $entry -m pip install -q pytest pytest-benchmark boto3 fastavro avro-python3 scikit-image pandas pyarrow==3.0.0 google-cloud-pubsub==2.1.0 google-cloud-bigtable==1.6.0 google-cloud-bigquery-storage==1.1.0 google-cloud-bigquery==2.3.1 google-cloud-storage==1.32.0 PyYAML==5.3.1 + $entry -m pip install -q pytest pytest-benchmark boto3 fastavro avro-python3 scikit-image pandas pyarrow==3.0.0 google-cloud-pubsub==2.1.0 google-cloud-bigtable==1.6.0 google-cloud-bigquery-storage==1.1.0 google-cloud-bigquery==2.3.1 google-cloud-storage==1.32.0 PyYAML==5.3.1 azure-storage-blob==12.8.1 (cd tests && $entry -m pytest --benchmark-disable -v --import-mode=append $(find . -type f \( -iname "test_*_v1.py" \))) (cd tests && $entry -m pytest --benchmark-disable -v --import-mode=append $(find . -type f \( -iname "test_*.py" ! \( -iname "test_*_v1.py" -o -iname "test_bigquery.py" \) \))) # GRPC and test_bigquery tests have to be executed separately because of https://github.com/grpc/grpc/issues/20034 diff --git a/tests/test_filesystem.py b/tests/test_filesystem.py index 8b5809bf7..625236abb 100644 --- a/tests/test_filesystem.py +++ b/tests/test_filesystem.py @@ -15,6 +15,7 @@ import os import posixpath +import sys import time from urllib.parse import urlparse @@ -22,14 +23,16 @@ import pytest import tensorflow as tf import tensorflow_io as tfio # pylint: disable=unused-import +from azure.storage.blob import ContainerClient -# pytestmark = pytest.mark.skipif( -# sys.platform in ("win32", "darwin"), -# reason="TODO emulator not setup properly on macOS/Windows yet", -# ) +pytestmark = pytest.mark.skipif( + sys.platform in ("win32", "darwin"), + reason="TODO emulator not setup properly on macOS/Windows yet", +) ROOT_PREFIX = f"tf-io-root-{int(time.time())}/" S3_URI = "s3" +AZ_URI = "az" def mock_patchs(monkeypatch, patchs): @@ -96,26 +99,80 @@ def mkdirs(path): path += "/" write(path, b"") - yield path_to, read, write, mkdirs, posixpath.join + yield S3_URI, path_to, read, write, mkdirs, posixpath.join + monkeypatch.undo() + + +@pytest.fixture(scope="module") +def az_fs(): + monkeypatch = pytest.MonkeyPatch() + container_name = os.environ.get("AZ_TEST_CONTAINER") + account = None + client = None + + # This means we are running against emulator. + if container_name is None: + monkeypatch.setenv("TF_AZURE_USE_DEV_STORAGE", "1") + container_name = f"tf-io-bucket-az-{int(time.time())}" + account = "devstoreaccount1" + conn_str = ( + "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;" + "AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq" + "/K1SZFPTOtr/KBHBeksoGMGw==;" + "BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;" + ) + client = ContainerClient.from_connection_string(conn_str, container_name) + client.create_container() + else: + # TODO(vnvo2409): Implement for testing against production scenario + pass + + client.upload_blob(ROOT_PREFIX, b"") + + def parse(path): + res = urlparse(path, scheme=AZ_URI, allow_fragments=False) + return res.path.split("/", 2)[2] + + def path_to(*args): + return f"{AZ_URI}://{account}/{container_name}/{posixpath.join(ROOT_PREFIX, *args)}" + + def read(path): + key_name = parse(path) + return client.download_blob(key_name).content_as_bytes() + + def write(path, body): + key_name = parse(path) + client.upload_blob(key_name, body) + + def mkdirs(_): + pass + + yield AZ_URI, path_to, read, write, mkdirs, posixpath.join monkeypatch.undo() @pytest.fixture -def fs(request, s3_fs): +def fs(request, s3_fs, az_fs): if request.param == S3_URI: return s3_fs + elif request.param == AZ_URI: + return az_fs -@pytest.mark.parametrize("fs, patchs", [(S3_URI, None)], indirect=["fs"]) +@pytest.mark.parametrize( + "fs, patchs", [(S3_URI, None), (AZ_URI, None)], indirect=["fs"] +) def test_init(fs, patchs, monkeypatch): - path_to, _, _, _, _ = fs + _, path_to, _, _, _, _ = fs mock_patchs(monkeypatch, patchs) assert tf.io.gfile.exists(path_to("")) is True -@pytest.mark.parametrize("fs, patchs", [(S3_URI, None)], indirect=["fs"]) +@pytest.mark.parametrize( + "fs, patchs", [(S3_URI, None), (AZ_URI, None)], indirect=["fs"] +) def test_io_read_file(fs, patchs, monkeypatch): - path_to, _, write, _, _ = fs + _, path_to, _, write, _, _ = fs mock_patchs(monkeypatch, patchs) fname = path_to("test_io_read_file") @@ -125,9 +182,11 @@ def test_io_read_file(fs, patchs, monkeypatch): assert tf.io.read_file(fname) == body -@pytest.mark.parametrize("fs, patchs", [(S3_URI, None)], indirect=["fs"]) +@pytest.mark.parametrize( + "fs, patchs", [(S3_URI, None), (AZ_URI, None)], indirect=["fs"] +) def test_io_write_file(fs, patchs, monkeypatch): - path_to, read, _, _, _ = fs + _, path_to, read, _, _, _ = fs mock_patchs(monkeypatch, patchs) fname = path_to("test_io_write_file") @@ -148,12 +207,13 @@ def test_io_write_file(fs, patchs, monkeypatch): lambda monkeypatch: monkeypatch.setattr( tf.io.gfile.GFile, "seekable", lambda _: False ), - ) + ), + (AZ_URI, None), ], indirect=["fs"], ) def test_gfile_GFile_readable(fs, patchs, monkeypatch): - path_to, _, write, _, _ = fs + uri, path_to, _, write, _, _ = fs mock_patchs(monkeypatch, patchs) fname = path_to("test_gfile_GFile_readable") @@ -169,10 +229,12 @@ def test_gfile_GFile_readable(fs, patchs, monkeypatch): assert file_read == body # Notfound - with pytest.raises(tf.errors.NotFoundError): - fname_not_found = fname + "_not_found" - with tf.io.gfile.GFile(fname_not_found, "rb") as f: - _ = f.read() + # TODO(vnvo2409): `az` should raise `tf.errors.NotFoundError`. + if uri != AZ_URI: + with pytest.raises(tf.errors.NotFoundError): + fname_not_found = fname + "_not_found" + with tf.io.gfile.GFile(fname_not_found, "rb") as f: + _ = f.read() # Read length with tf.io.gfile.GFile(fname, "rb") as f: @@ -211,9 +273,11 @@ def test_gfile_GFile_readable(fs, patchs, monkeypatch): assert file_read == body[seek_size : seek_size + read_length] -@pytest.mark.parametrize("fs, patchs", [(S3_URI, None)], indirect=["fs"]) +@pytest.mark.parametrize( + "fs, patchs", [(S3_URI, None), (AZ_URI, None)], indirect=["fs"] +) def test_gfile_GFile_writable(fs, patchs, monkeypatch): - path_to, read, _, _, _ = fs + uri, path_to, read, _, _, _ = fs mock_patchs(monkeypatch, patchs) fname = path_to("test_gfile_GFile_writable") @@ -230,15 +294,19 @@ def test_gfile_GFile_writable(fs, patchs, monkeypatch): assert read(fname) == body # Append - with tf.io.gfile.GFile(fname, "ab") as f: - f.write(base_body) - f.flush() - assert read(fname) == body + base_body + # TODO(vnvo2409): implement `az` appendable file. + if uri != AZ_URI: + with tf.io.gfile.GFile(fname, "ab") as f: + f.write(base_body) + f.flush() + assert read(fname) == body + base_body -@pytest.mark.parametrize("fs, patchs", [(S3_URI, None)], indirect=["fs"]) +@pytest.mark.parametrize( + "fs, patchs", [(S3_URI, None), (AZ_URI, None)], indirect=["fs"] +) def test_gfile_isdir(fs, patchs, monkeypatch): - path_to, _, write, mkdirs, join = fs + _, path_to, _, write, mkdirs, join = fs mock_patchs(monkeypatch, patchs) root_path = "test_gfile_isdir" @@ -252,9 +320,11 @@ def test_gfile_isdir(fs, patchs, monkeypatch): assert tf.io.gfile.isdir(fname) is False -@pytest.mark.parametrize("fs, patchs", [(S3_URI, None)], indirect=["fs"]) +@pytest.mark.parametrize( + "fs, patchs", [(S3_URI, None), (AZ_URI, None)], indirect=["fs"] +) def test_gfile_listdir(fs, patchs, monkeypatch): - path_to, _, write, mkdirs, join = fs + _, path_to, _, write, mkdirs, join = fs mock_patchs(monkeypatch, patchs) root_path = "test_gfile_listdir" @@ -276,9 +346,11 @@ def test_gfile_listdir(fs, patchs, monkeypatch): assert sorted(childrens) == sorted([join(dname, entry) for entry in entries]) -@pytest.mark.parametrize("fs, patchs", [(S3_URI, None)], indirect=["fs"]) +@pytest.mark.parametrize( + "fs, patchs", [(S3_URI, None), (AZ_URI, None)], indirect=["fs"] +) def test_gfile_makedirs(fs, patchs, monkeypatch): - path_to, _, write, _, join = fs + _, path_to, _, write, _, join = fs mock_patchs(monkeypatch, patchs) root_path = "test_gfile_makedirs/" @@ -292,9 +364,11 @@ def test_gfile_makedirs(fs, patchs, monkeypatch): assert tf.io.gfile.isdir(subdname) is True -@pytest.mark.parametrize("fs, patchs", [(S3_URI, None)], indirect=["fs"]) +@pytest.mark.parametrize( + "fs, patchs", [(S3_URI, None), (AZ_URI, None)], indirect=["fs"] +) def test_gfile_rmtree(fs, patchs, monkeypatch): - path_to, _, write, mkdirs, join = fs + _, path_to, _, write, mkdirs, join = fs mock_patchs(monkeypatch, patchs) num_entries = 3 @@ -313,9 +387,10 @@ def test_gfile_rmtree(fs, patchs, monkeypatch): assert [tf.io.gfile.exists(entry) for entry in trees] == [False] * num_entries +# TODO(vnvo2409): `az` copy operations causes an infinite loop. @pytest.mark.parametrize("fs, patchs", [(S3_URI, None)], indirect=["fs"]) def test_gfile_copy(fs, patchs, monkeypatch): - path_to, read, write, _, _ = fs + _, path_to, read, write, _, _ = fs mock_patchs(monkeypatch, patchs) src = path_to("test_gfile_copy_src") @@ -337,9 +412,11 @@ def test_gfile_copy(fs, patchs, monkeypatch): assert read(dst) == new_body -@pytest.mark.parametrize("fs, patchs", [(S3_URI, None)], indirect=["fs"]) +@pytest.mark.parametrize( + "fs, patchs", [(S3_URI, None), (AZ_URI, None)], indirect=["fs"] +) def test_gfile_glob(fs, patchs, monkeypatch): - path_to, _, write, _, join = fs + _, path_to, _, write, _, join = fs mock_patchs(monkeypatch, patchs) dname = path_to("test_gfile_glob/") From 7920ee60ac0042ef7a53a535861636b85f18f401 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=B5=20V=C4=83n=20Ngh=C4=A9a?= Date: Mon, 3 May 2021 08:52:53 +0200 Subject: [PATCH 3/4] bump `azurite` to `3.12.0` --- tests/test_azure/start_azure.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_azure/start_azure.sh b/tests/test_azure/start_azure.sh index af01e7db4..10cf15f75 100644 --- a/tests/test_azure/start_azure.sh +++ b/tests/test_azure/start_azure.sh @@ -2,7 +2,7 @@ set -e set -o pipefail -npm install azurite@2.7.0 +npm install azurite@3.12.0 echo starting azurite-blob $(npm bin)/azurite-blob & sleep 10 # Wait for storage emulator to start From f909faf1ece912b6a1683be231c481a8e8bd4bd9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=B5=20V=C4=83n=20Ngh=C4=A9a?= Date: Mon, 3 May 2021 10:00:05 +0200 Subject: [PATCH 4/4] add `test_filesystem` to `api` --- .github/workflows/api.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/api.yml b/.github/workflows/api.yml index 6f57660f1..c0d7692b4 100644 --- a/.github/workflows/api.yml +++ b/.github/workflows/api.yml @@ -50,6 +50,7 @@ jobs: python -m pytest -s -v tests/test_s3.py python -m pytest -s -v tests/test_azure.py python -m pytest -s -v tests/test_gcs.py + python -m pytest -s -v tests/test_filesystem.py linux: name: Linux ${{ matrix.python }} + ${{ matrix.version }} @@ -88,6 +89,7 @@ jobs: python -m pytest -s -v tests/test_s3.py python -m pytest -s -v tests/test_azure.py if [[ "${{ matrix.version }}" != "tf-nightly:tensorflow-io" ]]; then python -m pytest -s -v tests/test_gcs.py ; fi + python -m pytest -s -v tests/test_filesystem.py windows: name: Windows ${{ matrix.python }} + ${{ matrix.version }}