From 376fbda5da0d13721f51a0323de4e66204f745e4 Mon Sep 17 00:00:00 2001 From: Felipe Zipitria Date: Wed, 25 Nov 2020 12:01:27 -0300 Subject: [PATCH 01/13] feat(six): remove six Signed-off-by: Felipe Zipitria --- ftw/http.py | 61 ++++++++++++++++++-------------------------- ftw/logchecker.py | 8 +++--- ftw/pytest_plugin.py | 4 +-- ftw/ruleset.py | 5 ++-- ftw/testrunner.py | 5 ++-- 5 files changed, 34 insertions(+), 49 deletions(-) diff --git a/ftw/http.py b/ftw/http.py index a4b91d8..578467b 100644 --- a/ftw/http.py +++ b/ftw/http.py @@ -1,6 +1,5 @@ - import brotli -import io +from io import BytesIO import socket import ssl import errno @@ -13,29 +12,19 @@ import zlib import encodings from IPy import IP - -from six import BytesIO, PY2, ensure_binary, ensure_str, iteritems, \ - text_type -from six.moves import http_cookies +from http import cookies from . import errors +from . import util # Fallback to PROTOCOL_SSLv23 if PROTOCOL_TLS is not available. PROTOCOL_TLS = getattr(ssl, "PROTOCOL_TLS", ssl.PROTOCOL_SSLv23) -if PY2: - reload(sys) # pragma: no flakes - sys.setdefaultencoding('utf8') - escape_codec = 'string_escape' -else: - escape_codec = 'unicode_escape' - - class HttpResponse(object): def __init__(self, http_response, user_agent): - self.response = ensure_binary(http_response) + self.response = util.ensure_binary(http_response) # For testing purposes HTTPResponse might be called OOL try: self.dest_addr = user_agent.request_object.dest_addr @@ -139,7 +128,7 @@ def check_for_cookie(self, cookie): 'function': 'http.HttpResponse.check_for_cookie' }) try: - with io.open(psl_path, 'r', encoding='utf-8') as fo: + with open(psl_path, 'r', encoding='utf-8') as fo: for line in fo: if line[:2] == '//' or line[0] == ' ' or \ line[0].strip() == '': @@ -185,7 +174,7 @@ def process_response(self): Parses an HTTP response after an HTTP request is sent """ split_response = self.response.split(self.CRLF) - response_line = ensure_str(split_response[0]) + response_line = util.ensure_str(split_response[0]) response_headers = {} response_data = None data_line = None @@ -204,13 +193,13 @@ def process_response(self): 'header_rcvd': str(header), 'function': 'http.HttpResponse.process_response' }) - header = ensure_str(header[0]), ensure_str(header[1]) + header = util.ensure_str(header[0]), util.ensure_str(header[1]) response_headers[header[0].lower()] = header[1].lstrip() if 'set-cookie' in list(response_headers.keys()): try: - cookie = http_cookies.SimpleCookie() + cookie = cookies.SimpleCookie() cookie.load(response_headers['set-cookie']) - except http_cookies.CookieError as err: + except cookies.CookieError as err: raise errors.TestError( 'Error processing the cookie content into a SimpleCookie', { @@ -364,9 +353,9 @@ def build_request(self): if 'cookie' in list(self.request_object.headers.keys()): # Create a SimpleCookie out of our provided cookie try: - provided_cookie = http_cookies.SimpleCookie() + provided_cookie = cookies.SimpleCookie() provided_cookie.load(self.request_object.headers['cookie']) - except http_cookies.CookieError as err: + except cookies.CookieError as err: raise errors.TestError( 'Error processing the existing cookie into a ' 'SimpleCookie', @@ -390,16 +379,16 @@ def build_request(self): result_cookie[cookie_key] = \ cookie[cookie_key].value for key, value in iteritems(result_cookie): - cookie_value += (text_type(key) + '=' + - text_type(value) + '; ') + cookie_value += (str(key) + '=' + + str(value) + '; ') # Remove the trailing semicolon cookie_value = cookie_value[:-2] self.request_object.headers['cookie'] = cookie_value else: for cookie in available_cookies: for cookie_key, cookie_morsal in iteritems(cookie): - cookie_value += (text_type(cookie_key) + '=' + - text_type(cookie_morsal.coded_value) + + cookie_value += (str(cookie_key) + '=' + + str(cookie_morsal.coded_value) + '; ') # Remove the trailing semicolon cookie_value = cookie_value[:-2] @@ -410,7 +399,7 @@ def build_request(self): if self.request_object.headers != {}: for hname, hvalue in iteritems(self.request_object.headers): headers += text_type(hname) + ': ' + \ - text_type(hvalue) + self.CRLF + str(hvalue) + self.CRLF request = request.replace('#headers#', headers) # If we have data append it @@ -435,18 +424,18 @@ def build_request(self): if choice in possible_choices: encoding = choice try: - data = self.request_object.data.encode(encoding) - except UnicodeEncodeError as err: + data_bytes = self.request_object.data.encode(encoding, 'strict') + except UnicodeError as err: raise errors.TestError( 'Error encoding the data with the charset specified', { 'msg': str(err), 'Content-Type': str(self.request_object.headers['Content-Type']), - 'data': text_type(self.request_object.data), + 'data': str(self.request_object.data), 'function': 'http.HttpResponse.build_request' }) - request = request.replace('#data#', ensure_str(data)) + request = request.replace('#data#', util.ensure_str(data_bytes)) else: request = request.replace('#data#', '') # If we have a Raw Request we should use that instead @@ -457,15 +446,15 @@ def build_request(self): { 'function': 'http.HttpUA.build_request' }) - request = ensure_binary(self.request_object.raw_request) + request = self.request_object.raw_request.encode('utf-8', 'strict') # We do this regardless of magic if you want to send a literal # '\' 'r' or 'n' use encoded request. - request = request.decode(escape_codec) + request = request.decode('unicode_escape') if self.request_object.encoded_request is not None: request = base64.b64decode(self.request_object.encoded_request) - request = request.decode(escape_codec) + request = request.decode('unicode_escape') # if we have an Encoded request we should use that - self.request = ensure_binary(request) + self.request = request.encode('utf-8', 'strict') def get_response(self): """ @@ -486,7 +475,7 @@ def get_response(self): try: data = self.sock.recv(self.RECEIVE_BYTES) if data: - our_data.append(ensure_binary(data)) + our_data.append(util.ensure_binary(data)) begin = time.time() else: # Sleep for sometime to indicate a gap diff --git a/ftw/logchecker.py b/ftw/logchecker.py index c99ad83..7a0a990 100644 --- a/ftw/logchecker.py +++ b/ftw/logchecker.py @@ -1,9 +1,7 @@ -import abc -import six +from abc improt ABC, abstractmethod -@six.add_metaclass(abc.ABCMeta) -class LogChecker(): +class LogChecker(ABC): """ LogChecker is an abstract class that integrations with WAFs MUST implement. This class is used by the testrunner to test log lines against an expected @@ -17,7 +15,7 @@ def set_times(self, start, end): self.start = start self.end = end - @abc.abstractmethod + @abstractmethod def get_logs(self): """ MUST be implemented, MUST return an array of strings diff --git a/ftw/pytest_plugin.py b/ftw/pytest_plugin.py index 710e81f..e141d59 100644 --- a/ftw/pytest_plugin.py +++ b/ftw/pytest_plugin.py @@ -3,8 +3,8 @@ from . import util from .ruleset import Test -from six.moves.BaseHTTPServer import HTTPServer -from six.moves.SimpleHTTPServer import SimpleHTTPRequestHandler +from http.server import HTTPServer +from http.server import SimpleHTTPRequestHandler def get_testdata(rulesets): diff --git a/ftw/ruleset.py b/ftw/ruleset.py index 1a865f6..aefdfea 100644 --- a/ftw/ruleset.py +++ b/ftw/ruleset.py @@ -1,7 +1,6 @@ import re -from six import ensure_str -from six.moves.urllib.parse import parse_qsl, unquote, urlencode +from urllib.parse import parse_qsl, unquote, urlencode from . import errors @@ -122,7 +121,7 @@ def __init__(self, raw_request=None, if 'Content-Type' in list(headers.keys()): if headers['Content-Type'] == \ 'application/x-www-form-urlencoded' and stop_magic is False: - if ensure_str(unquote(self.data)) == self.data: + if unquote(self.data) == self.data: query_string = parse_qsl(self.data) if len(query_string) != 0: encoded_args = urlencode(query_string) diff --git a/ftw/testrunner.py b/ftw/testrunner.py index 12fbda0..217e6b5 100644 --- a/ftw/testrunner.py +++ b/ftw/testrunner.py @@ -4,7 +4,6 @@ import pytest import sqlite3 -from six import ensure_str from . import errors from . import http @@ -57,7 +56,7 @@ def test_response(self, response_object, regex): 'response_object': response_object, 'function': 'testrunner.TestRunner.test_response' }) - if regex.search(ensure_str(response_object.response)): + if regex.search(util.ensure_str(response_object.response)): assert True else: assert False @@ -67,7 +66,7 @@ def test_response_str(self, response, regex): Checks if the response response contains a regex specified in the output stage. It will assert that the regex is present. """ - if regex.search(ensure_str(response)): + if regex.search(util.ensure_str(response)): assert True else: assert False From 270a9c7af9384aa97dd8b2a80f848f4e1ebf82cb Mon Sep 17 00:00:00 2001 From: Felipe Zipitria Date: Wed, 25 Nov 2020 12:02:16 -0300 Subject: [PATCH 02/13] feat(util): add ensure helper functions Signed-off-by: Felipe Zipitria --- ftw/util.py | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/ftw/util.py b/ftw/util.py index ce7fdea..b7ed703 100644 --- a/ftw/util.py +++ b/ftw/util.py @@ -1,5 +1,4 @@ -import io import yaml import os import sqlite3 @@ -86,7 +85,7 @@ def extract_yaml(yaml_files): loaded_yaml = [] for yaml_file in yaml_files: try: - with io.open(yaml_file, encoding='utf-8') as fd: + with open(yaml_file, encoding='utf-8') as fd: loaded_yaml.append(yaml.safe_load(fd)) except IOError as e: print('Error reading file', yaml_file) @@ -98,3 +97,22 @@ def extract_yaml(yaml_files): print('General error') raise e return loaded_yaml + + +def ensure_str(s, encoding='utf-8', errors='strict'): + # Optimization: Fast return for the common case. + if type(s) is str: + return s + if isinstance(s, bytes): + return s.decode(encoding, errors) + elif not isinstance(s, (str, bytes)): + raise TypeError("not expecting type '%s'" % type(s)) + + +def ensure_binary(s, encoding='utf-8', errors='strict'): + if isinstance(s, bytes): + return s + if isinstance(s, str): + return s.encode(encoding, errors) + raise TypeError("not expecting type '%s'" % type(s)) + From cee9c0b1af326d018bfdd34f178e520f47a88dfe Mon Sep 17 00:00:00 2001 From: Felipe Zipitria Date: Wed, 25 Nov 2020 12:04:40 -0300 Subject: [PATCH 03/13] fix(build): remove six dependency Signed-off-by: Felipe Zipitria --- requirements.txt | 1 - setup.py | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index 6f40c81..a5d2a37 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,4 +3,3 @@ IPy==0.83 PyYAML==4.2b1 pytest==4.6 python-dateutil==2.6.0 -six==1.14.0 diff --git a/setup.py b/setup.py index 70cd745..9df9859 100644 --- a/setup.py +++ b/setup.py @@ -25,6 +25,5 @@ 'IPy==0.83', 'PyYAML==4.2b1', 'pytest==4.6', - 'python-dateutil==2.6.0', - 'six==1.14.0' + 'python-dateutil==2.6.0' ]) From aed586467d6c63b06b4fceff60d95bff688e8ad5 Mon Sep 17 00:00:00 2001 From: Felipe Zipitria Date: Wed, 25 Nov 2020 12:08:39 -0300 Subject: [PATCH 04/13] fix(typo): fixes typo in class Signed-off-by: Felipe Zipitria --- ftw/logchecker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ftw/logchecker.py b/ftw/logchecker.py index 7a0a990..3f3d557 100644 --- a/ftw/logchecker.py +++ b/ftw/logchecker.py @@ -1,4 +1,4 @@ -from abc improt ABC, abstractmethod +from abc import ABC, abstractmethod class LogChecker(ABC): From fb59dadbdf2d049a13c9ff34801b80aa11f6cf16 Mon Sep 17 00:00:00 2001 From: Felipe Zipitria Date: Wed, 25 Nov 2020 12:16:43 -0300 Subject: [PATCH 05/13] fix(util): use isinstance instead of type Signed-off-by: Felipe Zipitria --- ftw/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ftw/util.py b/ftw/util.py index b7ed703..b4687d5 100644 --- a/ftw/util.py +++ b/ftw/util.py @@ -101,7 +101,7 @@ def extract_yaml(yaml_files): def ensure_str(s, encoding='utf-8', errors='strict'): # Optimization: Fast return for the common case. - if type(s) is str: + if isinstance(s) is str: return s if isinstance(s, bytes): return s.decode(encoding, errors) From c19fad11a3d02c3488291e8a59f17701f79d11d4 Mon Sep 17 00:00:00 2001 From: Felipe Zipitria Date: Wed, 25 Nov 2020 12:17:14 -0300 Subject: [PATCH 06/13] fix(util): remove empty line Signed-off-by: Felipe Zipitria --- ftw/util.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ftw/util.py b/ftw/util.py index b4687d5..0560039 100644 --- a/ftw/util.py +++ b/ftw/util.py @@ -1,4 +1,3 @@ - import yaml import os import sqlite3 From ee2421ce7132dd1bd2152222cba87878763b85c4 Mon Sep 17 00:00:00 2001 From: Felipe Zipitria Date: Wed, 25 Nov 2020 12:27:00 -0300 Subject: [PATCH 07/13] fix(str): ensure we are using string Signed-off-by: Felipe Zipitria --- ftw/ruleset.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ftw/ruleset.py b/ftw/ruleset.py index aefdfea..7b250e1 100644 --- a/ftw/ruleset.py +++ b/ftw/ruleset.py @@ -3,6 +3,7 @@ from urllib.parse import parse_qsl, unquote, urlencode from . import errors +from . import util class Output(object): @@ -121,7 +122,7 @@ def __init__(self, raw_request=None, if 'Content-Type' in list(headers.keys()): if headers['Content-Type'] == \ 'application/x-www-form-urlencoded' and stop_magic is False: - if unquote(self.data) == self.data: + if util.ensure_str(unquote(self.data)) == self.data: query_string = parse_qsl(self.data) if len(query_string) != 0: encoded_args = urlencode(query_string) From f0380f763a8b39bf47b9ae16df412ebe4618b002 Mon Sep 17 00:00:00 2001 From: Felipe Zipitria Date: Wed, 25 Nov 2020 12:31:05 -0300 Subject: [PATCH 08/13] fix(util): fix introduced typo Signed-off-by: Felipe Zipitria --- ftw/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ftw/util.py b/ftw/util.py index 0560039..b58382e 100644 --- a/ftw/util.py +++ b/ftw/util.py @@ -100,7 +100,7 @@ def extract_yaml(yaml_files): def ensure_str(s, encoding='utf-8', errors='strict'): # Optimization: Fast return for the common case. - if isinstance(s) is str: + if isinstance(s, str): return s if isinstance(s, bytes): return s.decode(encoding, errors) From 678daf0b2ca529e4679ba8a0e1886420810c3532 Mon Sep 17 00:00:00 2001 From: Felipe Zipitria Date: Wed, 25 Nov 2020 12:34:54 -0300 Subject: [PATCH 09/13] feat(ci): test with multiple python versions Signed-off-by: Felipe Zipitria --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 56796e6..5ede2e0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,7 +9,7 @@ jobs: strategy: matrix: - python-version: [ '2.x', '3.6', '3.7', '3.8', '3.9' ] + python-version: [ '3.6', '3.7', '3.8', '3.9' ] steps: - name: Checkout repo From f33677eb0a441899b174ff2325ba3b50df1d1f28 Mon Sep 17 00:00:00 2001 From: Felipe Zipitria Date: Wed, 25 Nov 2020 12:41:28 -0300 Subject: [PATCH 10/13] fix(http): remove pending iteritems Signed-off-by: Felipe Zipitria --- ftw/http.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/ftw/http.py b/ftw/http.py index 578467b..4e8e354 100644 --- a/ftw/http.py +++ b/ftw/http.py @@ -1,4 +1,3 @@ -import brotli from io import BytesIO import socket import ssl @@ -11,6 +10,7 @@ import base64 import zlib import encodings +import brotli from IPy import IP from http import cookies @@ -366,11 +366,11 @@ def build_request(self): 'function': 'http.HttpResponse.build_request' }) result_cookie = {} - for cookie_key, cookie_morsal in iteritems(provided_cookie): + for cookie_key, cookie_morsal in provided_cookie: result_cookie[cookie_key] = \ provided_cookie[cookie_key].value for cookie in available_cookies: - for cookie_key, cookie_morsal in iteritems(cookie): + for cookie_key, cookie_morsal in cookie: if cookie_key in list(result_cookie.keys()): # we don't overwrite a user specified # cookie with a saved one @@ -378,7 +378,7 @@ def build_request(self): else: result_cookie[cookie_key] = \ cookie[cookie_key].value - for key, value in iteritems(result_cookie): + for key, value in result_cookie: cookie_value += (str(key) + '=' + str(value) + '; ') # Remove the trailing semicolon @@ -386,7 +386,7 @@ def build_request(self): self.request_object.headers['cookie'] = cookie_value else: for cookie in available_cookies: - for cookie_key, cookie_morsal in iteritems(cookie): + for cookie_key, cookie_morsal in cookie: cookie_value += (str(cookie_key) + '=' + str(cookie_morsal.coded_value) + '; ') @@ -397,8 +397,8 @@ def build_request(self): # Expand out our headers into a string headers = '' if self.request_object.headers != {}: - for hname, hvalue in iteritems(self.request_object.headers): - headers += text_type(hname) + ': ' + \ + for hname, hvalue in self.request_object.headers.items(): + headers += str(hname) + ': ' + \ str(hvalue) + self.CRLF request = request.replace('#headers#', headers) From b0b455c5b0ecf2228271b6658089ab40a4c2c8ff Mon Sep 17 00:00:00 2001 From: Felipe Zipitria Date: Wed, 25 Nov 2020 13:52:07 -0300 Subject: [PATCH 11/13] fix(http): add additional items from cookies Signed-off-by: Felipe Zipitria --- ftw/http.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ftw/http.py b/ftw/http.py index 4e8e354..55b8f57 100644 --- a/ftw/http.py +++ b/ftw/http.py @@ -366,7 +366,7 @@ def build_request(self): 'function': 'http.HttpResponse.build_request' }) result_cookie = {} - for cookie_key, cookie_morsal in provided_cookie: + for cookie_key, cookie_morsal in list(provided_cookie.items()): result_cookie[cookie_key] = \ provided_cookie[cookie_key].value for cookie in available_cookies: @@ -378,7 +378,7 @@ def build_request(self): else: result_cookie[cookie_key] = \ cookie[cookie_key].value - for key, value in result_cookie: + for key, value in list(result_cookie.items()): cookie_value += (str(key) + '=' + str(value) + '; ') # Remove the trailing semicolon @@ -386,7 +386,7 @@ def build_request(self): self.request_object.headers['cookie'] = cookie_value else: for cookie in available_cookies: - for cookie_key, cookie_morsal in cookie: + for cookie_key, cookie_morsal in list(cookie.items()): cookie_value += (str(cookie_key) + '=' + str(cookie_morsal.coded_value) + '; ') From c8a12126f5602721e2dc3ba0ccb93efee78a7bab Mon Sep 17 00:00:00 2001 From: Felipe Zipitria Date: Wed, 25 Nov 2020 18:40:35 -0300 Subject: [PATCH 12/13] fix(style): split lines Signed-off-by: Felipe Zipitria --- ftw/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ftw/util.py b/ftw/util.py index b58382e..176c447 100644 --- a/ftw/util.py +++ b/ftw/util.py @@ -1,7 +1,7 @@ -import yaml import os import sqlite3 from glob import glob +import yaml from . import ruleset From 96f0b38d669f304313193b1722a10da053823465 Mon Sep 17 00:00:00 2001 From: Felipe Zipitria Date: Thu, 26 Nov 2020 13:35:27 -0300 Subject: [PATCH 13/13] fix(flake8): remove spurious empty spaces Signed-off-by: Felipe Zipitria --- ftw/http.py | 3 ++- ftw/util.py | 5 ++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ftw/http.py b/ftw/http.py index 55b8f57..31e6f27 100644 --- a/ftw/http.py +++ b/ftw/http.py @@ -424,7 +424,8 @@ def build_request(self): if choice in possible_choices: encoding = choice try: - data_bytes = self.request_object.data.encode(encoding, 'strict') + data_bytes = \ + self.request_object.data.encode(encoding, 'strict') except UnicodeError as err: raise errors.TestError( 'Error encoding the data with the charset specified', diff --git a/ftw/util.py b/ftw/util.py index 176c447..7c4ef09 100644 --- a/ftw/util.py +++ b/ftw/util.py @@ -96,8 +96,8 @@ def extract_yaml(yaml_files): print('General error') raise e return loaded_yaml - - + + def ensure_str(s, encoding='utf-8', errors='strict'): # Optimization: Fast return for the common case. if isinstance(s, str): @@ -114,4 +114,3 @@ def ensure_binary(s, encoding='utf-8', errors='strict'): if isinstance(s, str): return s.encode(encoding, errors) raise TypeError("not expecting type '%s'" % type(s)) -