From f466aaadac944166c24b2326575dbad47f242fc9 Mon Sep 17 00:00:00 2001
From: Fokko Driesprong <fokko@apache.org>
Date: Wed, 13 May 2020 19:34:13 +0200
Subject: [PATCH 1/3] Enable mypy on the repository

---
 .gitignore                                          |  3 +++
 python/iceberg/api/expressions/expression_parser.py |  2 +-
 python/iceberg/api/transforms/bucket.py             |  2 +-
 python/iceberg/api/types/type_util.py               |  6 +++---
 python/iceberg/core/avro/avro_to_iceberg.py         |  2 +-
 .../iceberg/core/base_metastore_table_operations.py |  2 +-
 python/iceberg/core/base_table_scan.py              |  4 ++--
 python/iceberg/core/filesystem/s3_filesystem.py     | 10 +++++-----
 python/iceberg/core/manifest_list_writer.py         |  2 +-
 python/iceberg/core/manifest_reader.py              |  6 +++---
 python/iceberg/core/scan_summary.py                 |  2 +-
 python/iceberg/hive/hive_tables.py                  |  2 +-
 python/tox.ini                                      | 13 ++++++++++---
 13 files changed, 33 insertions(+), 23 deletions(-)

diff --git a/.gitignore b/.gitignore
index 13767d622f82..f1e459c8cb0a 100644
--- a/.gitignore
+++ b/.gitignore
@@ -38,3 +38,6 @@ bin/
 
 # Hive/metastore files
 metastore_db/
+
+# Python stuff
+python/.mypy_cache/
diff --git a/python/iceberg/api/expressions/expression_parser.py b/python/iceberg/api/expressions/expression_parser.py
index 0ffde4e23e3b..a26e46a24a66 100644
--- a/python/iceberg/api/expressions/expression_parser.py
+++ b/python/iceberg/api/expressions/expression_parser.py
@@ -19,7 +19,7 @@
 
 import logging
 
-from pyparsing import (
+from pyparsing import (  # type: ignore
     alphanums,
     alphas,
     CaselessKeyword,
diff --git a/python/iceberg/api/transforms/bucket.py b/python/iceberg/api/transforms/bucket.py
index c1672da4a4eb..678df9471be8 100644
--- a/python/iceberg/api/transforms/bucket.py
+++ b/python/iceberg/api/transforms/bucket.py
@@ -19,7 +19,7 @@
 import struct
 import sys
 
-import mmh3
+import mmh3  # type: ignore
 
 from .transform import Transform
 from .transform_util import TransformUtil
diff --git a/python/iceberg/api/types/type_util.py b/python/iceberg/api/types/type_util.py
index ca52a7db9c6f..8bd14a7a2108 100644
--- a/python/iceberg/api/types/type_util.py
+++ b/python/iceberg/api/types/type_util.py
@@ -16,6 +16,7 @@
 # under the License.
 
 import math
+from typing import List
 
 from .type import (Type,
                    TypeID)
@@ -238,7 +239,6 @@ def get(self):
         return self.visitor.field(self.field, VisitFuture(self.field.type, self.visitor).get)
 
 
-@staticmethod
 def decimal_required_bytes(precision):
     if precision < 0 or precision > 40:
         raise RuntimeError("Unsupported decimal precision: %s" % precision)
@@ -451,11 +451,11 @@ def write_compatibility_errors(read_schema, write_schema):
     def read_compatibility_errors(read_schema, write_schema):
         visit(write_schema, CheckCompatibility(read_schema, False))
 
-    NO_ERRORS = []
+    NO_ERRORS: List[str] = []
 
     def __init__(self, schema, check_ordering):
         self.schema = schema
-        self.check_ordering
+        self.check_ordering = check_ordering
         self.current_type = None
 
     def schema(self, schema, struct_result):
diff --git a/python/iceberg/core/avro/avro_to_iceberg.py b/python/iceberg/core/avro/avro_to_iceberg.py
index 14dc9ce9d468..5a5db0f6520d 100644
--- a/python/iceberg/core/avro/avro_to_iceberg.py
+++ b/python/iceberg/core/avro/avro_to_iceberg.py
@@ -15,7 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-import fastavro
+import fastavro  # type: ignore
 from iceberg.api import Schema
 from iceberg.api.types import (BinaryType,
                                BooleanType,
diff --git a/python/iceberg/core/base_metastore_table_operations.py b/python/iceberg/core/base_metastore_table_operations.py
index dc4594a18243..a8400320f9d5 100644
--- a/python/iceberg/core/base_metastore_table_operations.py
+++ b/python/iceberg/core/base_metastore_table_operations.py
@@ -18,7 +18,7 @@
 import logging
 import uuid
 
-from retrying import retry
+from retrying import retry  # type: ignore
 
 from .table_metadata_parser import TableMetadataParser
 from .table_operations import TableOperations
diff --git a/python/iceberg/core/base_table_scan.py b/python/iceberg/core/base_table_scan.py
index d205364aad15..e64038d3136a 100644
--- a/python/iceberg/core/base_table_scan.py
+++ b/python/iceberg/core/base_table_scan.py
@@ -34,9 +34,9 @@
 
 class BaseTableScan(CloseableGroup, TableScan):
     DATE_FORMAT = "%Y-%m-%d %H:%M:%S.%f"
-    SNAPSHOT_COLUMNS = ["snapshot_id", "file_path", "file_ordinal", "file_format", "block_size_in_bytes",
+    SNAPSHOT_COLUMNS = ("snapshot_id", "file_path", "file_ordinal", "file_format", "block_size_in_bytes",
                         "file_size_in_bytes", "record_count", "partition", "value_counts", "null_value_counts",
-                        "lower_bounds", "upper_bounds"]
+                        "lower_bounds", "upper_bounds")
 
     def new_refined_scan(self, ops, table, schema, snapshot_id, row_filter,
                          case_sensitive, selected_columns, options, minused_cols):
diff --git a/python/iceberg/core/filesystem/s3_filesystem.py b/python/iceberg/core/filesystem/s3_filesystem.py
index a8ac9d4cca25..0f6282170d7d 100644
--- a/python/iceberg/core/filesystem/s3_filesystem.py
+++ b/python/iceberg/core/filesystem/s3_filesystem.py
@@ -21,11 +21,11 @@
 import time
 from urllib.parse import urlparse
 
-import boto3
-from botocore.credentials import RefreshableCredentials
-from botocore.exceptions import ClientError
-from botocore.session import get_session
-from retrying import retry
+import boto3  # type: ignore
+from botocore.credentials import RefreshableCredentials  # type: ignore
+from botocore.exceptions import ClientError  # type: ignore
+from botocore.session import get_session  # type: ignore
+from retrying import retry  # type: ignore
 
 from .file_status import FileStatus
 from .file_system import FileSystem
diff --git a/python/iceberg/core/manifest_list_writer.py b/python/iceberg/core/manifest_list_writer.py
index b3078edf537e..9630f3478e03 100644
--- a/python/iceberg/core/manifest_list_writer.py
+++ b/python/iceberg/core/manifest_list_writer.py
@@ -17,7 +17,7 @@
 
 import json
 
-from fastavro import parse_schema, writer
+from fastavro import parse_schema, writer  # type: ignore
 from iceberg.api import ManifestFile
 from iceberg.api.io import FileAppender
 from iceberg.core import GenericManifestFile
diff --git a/python/iceberg/core/manifest_reader.py b/python/iceberg/core/manifest_reader.py
index 6a4581293a6f..b1147735db21 100644
--- a/python/iceberg/core/manifest_reader.py
+++ b/python/iceberg/core/manifest_reader.py
@@ -17,7 +17,7 @@
 
 import logging
 
-import fastavro
+import fastavro  # type: ignore
 from iceberg.api import FileFormat, Filterable
 from iceberg.api.expressions import Expressions, inclusive
 from iceberg.api.io import CloseableGroup
@@ -33,8 +33,8 @@
 
 
 class ManifestReader(CloseableGroup, Filterable):
-    ALL_COLUMNS = ["*"]
-    CHANGE_COLUMNS = ["file_path", "file_format", "partition", "record_count", "file_size_in_bytes"]
+    ALL_COLUMNS = ("*",)
+    CHANGE_COLUMNS = ("file_path", "file_format", "partition", "record_count", "file_size_in_bytes")
 
     @staticmethod
     def read(file, spec_lookup=None):
diff --git a/python/iceberg/core/scan_summary.py b/python/iceberg/core/scan_summary.py
index cef872f7ea9a..330338608139 100644
--- a/python/iceberg/core/scan_summary.py
+++ b/python/iceberg/core/scan_summary.py
@@ -22,7 +22,7 @@
 from iceberg.api.expressions import Expressions, Literal, Operation, UnboundPredicate
 from iceberg.api.types import TimestampType
 
-from .manifest_group import ManifestGroup
+from .manifest_group import ManifestGroup  # type: ignore
 from .util import str_as_bool
 
 TIMESTAMP_RANGE_MAP = {Operation.LT: lambda min_val, max_val, val: (min_val, val - 1 if val - 1 < max_val else max_val),
diff --git a/python/iceberg/hive/hive_tables.py b/python/iceberg/hive/hive_tables.py
index 42d734c46898..4d48ce2de6c1 100644
--- a/python/iceberg/hive/hive_tables.py
+++ b/python/iceberg/hive/hive_tables.py
@@ -18,7 +18,7 @@
 #
 
 
-from hmsclient import hmsclient
+from hmsclient import hmsclient  # type: ignore
 from iceberg.core import BaseMetastoreTables
 
 from .hive_table_operations import HiveTableOperations
diff --git a/python/tox.ini b/python/tox.ini
index 9c49b0bd33cc..66453c3035d0 100644
--- a/python/tox.ini
+++ b/python/tox.ini
@@ -39,9 +39,11 @@ deps =
     .
     {[testenv:flake8]deps}
     {[testenv:bandit]deps}
+    {[testenv:mypy]deps}
 commands =
     {[testenv:flake8]commands}
     {[testenv:bandit]commands}
+    {[testenv:mypy]commands}
 
 [testenv:flake8]
 basepython = python3
@@ -53,6 +55,14 @@ deps =
 commands =
     flake8 iceberg setup.py tests
 
+[testenv:mypy]
+basepython = python3
+skip_install = true
+deps =
+    mypy
+commands =
+    mypy iceberg/
+
 [testenv:bandit]
 basepython = python3
 skip_install = true
@@ -77,9 +87,6 @@ commands =
 # commands =
 #     python -m http.server {posargs}
 
-[bandit]
-skips = B104
-
 [flake8]
 ignore = E501,W503
 exclude =

From 4356d1ad5abedf73958e6a45e8941d38e33f8f60 Mon Sep 17 00:00:00 2001
From: Fokko Driesprong <fokko@apache.org>
Date: Wed, 13 May 2020 21:04:06 +0200
Subject: [PATCH 2/3] Add type annotation on field method

---
 python/iceberg/api/types/type_util.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/python/iceberg/api/types/type_util.py b/python/iceberg/api/types/type_util.py
index 8bd14a7a2108..68f0f595acf3 100644
--- a/python/iceberg/api/types/type_util.py
+++ b/python/iceberg/api/types/type_util.py
@@ -498,10 +498,10 @@ def struct(self, struct, field_results):
 
         return errors
 
-    def field(self, field, field_result):
+    def field(self, field, field_result) -> List[str]:
         struct = self.current_type.as_struct_type()
         curr_field = struct.field(field.field_id)
-        errors = list()
+        errors = []
 
         if curr_field is None:
             if not field.is_optional:

From 6b8a1f126c06de4bfcd0bc037784ba90306809e0 Mon Sep 17 00:00:00 2001
From: Fokko Driesprong <fokko@apache.org>
Date: Thu, 21 May 2020 11:57:39 +0200
Subject: [PATCH 3/3] Add --ignore-missing-imports

---
 python/iceberg/api/expressions/expression_parser.py    |  2 +-
 python/iceberg/api/transforms/bucket.py                |  2 +-
 python/iceberg/core/avro/avro_to_iceberg.py            |  2 +-
 python/iceberg/core/base_metastore_table_operations.py |  2 +-
 python/iceberg/core/filesystem/s3_filesystem.py        | 10 +++++-----
 python/iceberg/core/manifest_list_writer.py            |  2 +-
 python/iceberg/core/manifest_reader.py                 |  2 +-
 python/iceberg/core/scan_summary.py                    |  2 +-
 python/iceberg/hive/hive_tables.py                     |  2 +-
 python/tox.ini                                         |  2 +-
 10 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/python/iceberg/api/expressions/expression_parser.py b/python/iceberg/api/expressions/expression_parser.py
index a26e46a24a66..0ffde4e23e3b 100644
--- a/python/iceberg/api/expressions/expression_parser.py
+++ b/python/iceberg/api/expressions/expression_parser.py
@@ -19,7 +19,7 @@
 
 import logging
 
-from pyparsing import (  # type: ignore
+from pyparsing import (
     alphanums,
     alphas,
     CaselessKeyword,
diff --git a/python/iceberg/api/transforms/bucket.py b/python/iceberg/api/transforms/bucket.py
index 678df9471be8..c1672da4a4eb 100644
--- a/python/iceberg/api/transforms/bucket.py
+++ b/python/iceberg/api/transforms/bucket.py
@@ -19,7 +19,7 @@
 import struct
 import sys
 
-import mmh3  # type: ignore
+import mmh3
 
 from .transform import Transform
 from .transform_util import TransformUtil
diff --git a/python/iceberg/core/avro/avro_to_iceberg.py b/python/iceberg/core/avro/avro_to_iceberg.py
index 5a5db0f6520d..14dc9ce9d468 100644
--- a/python/iceberg/core/avro/avro_to_iceberg.py
+++ b/python/iceberg/core/avro/avro_to_iceberg.py
@@ -15,7 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-import fastavro  # type: ignore
+import fastavro
 from iceberg.api import Schema
 from iceberg.api.types import (BinaryType,
                                BooleanType,
diff --git a/python/iceberg/core/base_metastore_table_operations.py b/python/iceberg/core/base_metastore_table_operations.py
index a8400320f9d5..dc4594a18243 100644
--- a/python/iceberg/core/base_metastore_table_operations.py
+++ b/python/iceberg/core/base_metastore_table_operations.py
@@ -18,7 +18,7 @@
 import logging
 import uuid
 
-from retrying import retry  # type: ignore
+from retrying import retry
 
 from .table_metadata_parser import TableMetadataParser
 from .table_operations import TableOperations
diff --git a/python/iceberg/core/filesystem/s3_filesystem.py b/python/iceberg/core/filesystem/s3_filesystem.py
index 0f6282170d7d..a8ac9d4cca25 100644
--- a/python/iceberg/core/filesystem/s3_filesystem.py
+++ b/python/iceberg/core/filesystem/s3_filesystem.py
@@ -21,11 +21,11 @@
 import time
 from urllib.parse import urlparse
 
-import boto3  # type: ignore
-from botocore.credentials import RefreshableCredentials  # type: ignore
-from botocore.exceptions import ClientError  # type: ignore
-from botocore.session import get_session  # type: ignore
-from retrying import retry  # type: ignore
+import boto3
+from botocore.credentials import RefreshableCredentials
+from botocore.exceptions import ClientError
+from botocore.session import get_session
+from retrying import retry
 
 from .file_status import FileStatus
 from .file_system import FileSystem
diff --git a/python/iceberg/core/manifest_list_writer.py b/python/iceberg/core/manifest_list_writer.py
index 9630f3478e03..b3078edf537e 100644
--- a/python/iceberg/core/manifest_list_writer.py
+++ b/python/iceberg/core/manifest_list_writer.py
@@ -17,7 +17,7 @@
 
 import json
 
-from fastavro import parse_schema, writer  # type: ignore
+from fastavro import parse_schema, writer
 from iceberg.api import ManifestFile
 from iceberg.api.io import FileAppender
 from iceberg.core import GenericManifestFile
diff --git a/python/iceberg/core/manifest_reader.py b/python/iceberg/core/manifest_reader.py
index b1147735db21..e648684f29fe 100644
--- a/python/iceberg/core/manifest_reader.py
+++ b/python/iceberg/core/manifest_reader.py
@@ -17,7 +17,7 @@
 
 import logging
 
-import fastavro  # type: ignore
+import fastavro
 from iceberg.api import FileFormat, Filterable
 from iceberg.api.expressions import Expressions, inclusive
 from iceberg.api.io import CloseableGroup
diff --git a/python/iceberg/core/scan_summary.py b/python/iceberg/core/scan_summary.py
index 330338608139..cef872f7ea9a 100644
--- a/python/iceberg/core/scan_summary.py
+++ b/python/iceberg/core/scan_summary.py
@@ -22,7 +22,7 @@
 from iceberg.api.expressions import Expressions, Literal, Operation, UnboundPredicate
 from iceberg.api.types import TimestampType
 
-from .manifest_group import ManifestGroup  # type: ignore
+from .manifest_group import ManifestGroup
 from .util import str_as_bool
 
 TIMESTAMP_RANGE_MAP = {Operation.LT: lambda min_val, max_val, val: (min_val, val - 1 if val - 1 < max_val else max_val),
diff --git a/python/iceberg/hive/hive_tables.py b/python/iceberg/hive/hive_tables.py
index 4d48ce2de6c1..42d734c46898 100644
--- a/python/iceberg/hive/hive_tables.py
+++ b/python/iceberg/hive/hive_tables.py
@@ -18,7 +18,7 @@
 #
 
 
-from hmsclient import hmsclient  # type: ignore
+from hmsclient import hmsclient
 from iceberg.core import BaseMetastoreTables
 
 from .hive_table_operations import HiveTableOperations
diff --git a/python/tox.ini b/python/tox.ini
index 66453c3035d0..b011511130b2 100644
--- a/python/tox.ini
+++ b/python/tox.ini
@@ -61,7 +61,7 @@ skip_install = true
 deps =
     mypy
 commands =
-    mypy iceberg/
+    mypy --ignore-missing-imports iceberg/
 
 [testenv:bandit]
 basepython = python3