From 6cbb156a01e8dcb700cfff38cc0b1bbda901baea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20St=C3=A1rek?= Date: Thu, 24 Oct 2019 15:57:28 +0200 Subject: [PATCH 1/4] Adding query attributes --- fuzzer/src/blocks_generator.py | 2 +- fuzzer/src/request_build_helper.py | 46 +++++++++++++++++++++--------- parser/Models/UriAttribute.cs | 1 + parser/Parser/AttributeParser.cs | 3 +- 4 files changed, 37 insertions(+), 15 deletions(-) diff --git a/fuzzer/src/blocks_generator.py b/fuzzer/src/blocks_generator.py index 8954028..ed47cae 100644 --- a/fuzzer/src/blocks_generator.py +++ b/fuzzer/src/blocks_generator.py @@ -89,7 +89,7 @@ def _generate_content_body(is_body_json, json_decoder, body_string_example, fuzz def _generate_http_header(request, endpoint, fuzzable): s_static(request["Method"].upper() + " ") - RequestBuildHelper.generate_uri(endpoint["Uri"], request["UriAttributes"], ConfigurationManager.config, fuzzable) + RequestBuildHelper.generate_uri(endpoint["Uri"], request["UriAttributes"], fuzzable) s_static(" HTTP/1.1\r\n") RequestBuildHelper.generate_headers(ConfigurationManager.config) s_static("\r\n\r\n") diff --git a/fuzzer/src/request_build_helper.py b/fuzzer/src/request_build_helper.py index 6e2be4a..d8e8e81 100644 --- a/fuzzer/src/request_build_helper.py +++ b/fuzzer/src/request_build_helper.py @@ -1,8 +1,10 @@ import json -from boofuzz import s_static, s_size +from typing import List +from boofuzz import s_static, s_size, s_render from fuzz_payloads import s_http_string, s_http_number, s_http_boolean from encodings_helper import EncodingTypes from parameter import Parameter +from configuration_manager import ConfigurationManager class RequestBuildHelper(object): @@ -34,10 +36,12 @@ class RequestBuildHelper(object): return headers is not None and header_name in headers @staticmethod - def generate_uri(uri, uri_parameters, config, fuzzable=False): - fixed_attributes = config["fixed_url_attributes"] if "fixed_url_attributes" in config else None + def generate_uri(uri, uri_parameters, fuzzable=False): id_generator = _unique_uri_attribute_id() + already_used_parameters: List[str] = [] + + # 1] Generate URI as it is in payloads file while True: try: # Find first not yet found parameter, if there is one @@ -48,24 +52,40 @@ class RequestBuildHelper(object): index = uri.index("}") parameter_name = uri[0:index] - parameter: Parameter = RequestBuildHelper._get_parameter(parameter_name, fixed_attributes, uri_parameters) - name = "URI attribute, default value: " + parameter.value + ", id: " + next(id_generator) - is_part_fuzzable = fuzzable and not parameter.is_from_config - - if parameter.data_type and (parameter.data_type == 'integer' or parameter.data_type == 'number'): - s_http_number(parameter.value, fuzzable=is_part_fuzzable, encoding=EncodingTypes.urlencoded, name=name) - elif parameter.data_type and parameter.data_type == 'string': - s_http_boolean(parameter.value, fuzzable=is_part_fuzzable, encoding=EncodingTypes.urlencoded, name=name) - else: - s_http_string(parameter.value, fuzzable=is_part_fuzzable, encoding=EncodingTypes.urlencoded, name=name) + RequestBuildHelper._append_parameter(parameter_name, id_generator, uri_parameters, fuzzable) uri = uri[index + 1:] + already_used_parameters.append(parameter_name) except ValueError: if len(uri) > 0: name = "URI attribute, default value: " + uri + ", id: " + next(id_generator) s_http_string(uri, fuzzable=False, encoding=EncodingTypes.ascii, name=name) break + # 2] Append another URI attributes + for uri_parameter in uri_parameters: + parameter_name = uri_parameter["Name"] + if parameter_name not in already_used_parameters and uri_parameter["Location"] == "Query": + prefix = "?" if "?" not in s_render() else "&" + name = "URI attribute, default value: " + uri + ", id: " + next(id_generator) + s_http_string(prefix + parameter_name + "=", fuzzable=False, encoding=EncodingTypes.ascii, name=name) + RequestBuildHelper._append_parameter(parameter_name, id_generator, uri_parameters, fuzzable) + + @staticmethod + def _append_parameter(parameter_name, id_generator, uri_parameters, fuzzable): + fixed_attributes = ConfigurationManager.config["fixed_url_attributes"] if "fixed_url_attributes" in ConfigurationManager.config else None + + parameter: Parameter = RequestBuildHelper._get_parameter(parameter_name, fixed_attributes, uri_parameters) + name = "URI attribute, default value: " + parameter.value + ", id: " + next(id_generator) + is_part_fuzzable = fuzzable and not parameter.is_from_config + + if parameter.data_type and (parameter.data_type == 'integer' or parameter.data_type == 'number'): + s_http_number(parameter.value, fuzzable=is_part_fuzzable, encoding=EncodingTypes.urlencoded, name=name) + elif parameter.data_type and parameter.data_type == 'string': + s_http_boolean(parameter.value, fuzzable=is_part_fuzzable, encoding=EncodingTypes.urlencoded, name=name) + else: + s_http_string(parameter.value, fuzzable=is_part_fuzzable, encoding=EncodingTypes.urlencoded, name=name) + # Getting parameter value from these sources (ordered): # 1] Fixed attributes from config # 2] Example value from documentation diff --git a/parser/Models/UriAttribute.cs b/parser/Models/UriAttribute.cs index 36809e4..a74a143 100644 --- a/parser/Models/UriAttribute.cs +++ b/parser/Models/UriAttribute.cs @@ -10,6 +10,7 @@ namespace Models public string Type { get; set; } public string Format { get; set; } + public string Location { get; set; } public UriAttribute(string name, bool required) { diff --git a/parser/Parser/AttributeParser.cs b/parser/Parser/AttributeParser.cs index b1bd001..8d1ce4f 100644 --- a/parser/Parser/AttributeParser.cs +++ b/parser/Parser/AttributeParser.cs @@ -21,7 +21,8 @@ namespace Parser ContentParser.GetSingleExample(parameter.Schema?.Example) ?? PrimitiveDataTypeExampleGenerator.GenerateExampleValueByType(parameter.Schema.Type, parameter.Schema.Format), Type = parameter.Schema.Type, - Format = parameter.Schema.Format + Format = parameter.Schema.Format, + Location = parameter.In == ParameterLocation.Path ? "Path" : "Query" }; return attribute; } From 5d041957767757b2b9ab6d4c6bf6d3e430d8770c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20St=C3=A1rek?= Date: Mon, 28 Oct 2019 10:59:52 +0100 Subject: [PATCH 2/4] Decode boofuzz bytes into ascii The whole header part is forced to be ascii, so the decode should be quite safe. --- fuzzer/src/request_build_helper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fuzzer/src/request_build_helper.py b/fuzzer/src/request_build_helper.py index d8e8e81..3da60b2 100644 --- a/fuzzer/src/request_build_helper.py +++ b/fuzzer/src/request_build_helper.py @@ -66,7 +66,7 @@ class RequestBuildHelper(object): for uri_parameter in uri_parameters: parameter_name = uri_parameter["Name"] if parameter_name not in already_used_parameters and uri_parameter["Location"] == "Query": - prefix = "?" if "?" not in s_render() else "&" + prefix = "?" if "?" not in s_render().decode('ascii', 'ignore') else "&" name = "URI attribute, default value: " + uri + ", id: " + next(id_generator) s_http_string(prefix + parameter_name + "=", fuzzable=False, encoding=EncodingTypes.ascii, name=name) RequestBuildHelper._append_parameter(parameter_name, id_generator, uri_parameters, fuzzable) From 01c9afaacaf7792670509c7f46e743ab35e680ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20St=C3=A1rek?= Date: Mon, 28 Oct 2019 12:01:50 +0100 Subject: [PATCH 3/4] Added unit tests for both parser and fuzzer --- .travis.yml | 1 + .../unit_tests/request_build_helper_tests.py | 86 +++++++++++++++++++ parser/Parser.Tests/AttributeParserTests.cs | 28 ++++++ 3 files changed, 115 insertions(+) create mode 100644 fuzzer/src/unit_tests/request_build_helper_tests.py diff --git a/.travis.yml b/.travis.yml index 62d1bd8..3689f4b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,4 +18,5 @@ script: - cd ~/build/ysoftdevs/wapifuzz/parser/ && dotnet restore && dotnet test - cd ~/build/ysoftdevs/wapifuzz/fuzzer/src/ && python3 -m unittest unit_tests.fuzzing_json_decoder_tests - cd ~/build/ysoftdevs/wapifuzz/fuzzer/src/ && python3 -m unittest unit_tests.json_schema_parser_tests + - cd ~/build/ysoftdevs/wapifuzz/fuzzer/src/ && python3 -m unittest unit_tests.request_build_helper_tests - cd ~/build/ysoftdevs/wapifuzz/tests/ && chmod +x run_tests.sh && travis_wait ./run_tests.sh diff --git a/fuzzer/src/unit_tests/request_build_helper_tests.py b/fuzzer/src/unit_tests/request_build_helper_tests.py new file mode 100644 index 0000000..5237684 --- /dev/null +++ b/fuzzer/src/unit_tests/request_build_helper_tests.py @@ -0,0 +1,86 @@ +import unittest +from request_build_helper import RequestBuildHelper +from boofuzz import * +from configuration_manager import ConfigurationManager + + +class RequestBuilderHelperTests(unittest.TestCase): + def setUp(self): + # Just init block for boofuzz + s_initialize(self.id()) + + ConfigurationManager.config = [] + + def test_generate_simple_uri_without_parameters(self): + uri_parameters = [] + base_uri = '/api/endpoint' + + RequestBuildHelper.generate_uri(base_uri, uri_parameters) + + uri = s_render().decode('utf8', 'ignore') + self.assertEqual(base_uri, uri) + + def test_generate_uri_path_parameter_without_documentation(self): + uri_parameters = [] + + RequestBuildHelper.generate_uri('/api/endpoint/{id}', uri_parameters) + + uri = s_render().decode('utf8', 'ignore') + self.assertEqual('/api/endpoint/attribute', uri) + + def test_generate_uri_path_parameter_with_fixed_config_value(self): + uri_parameters = [] + ConfigurationManager.config = { + "fixed_url_attributes": { + "id": "20" + } + } + + RequestBuildHelper.generate_uri('/api/endpoint/{id}', uri_parameters) + + uri = s_render().decode('utf8', 'ignore') + self.assertEqual('/api/endpoint/20', uri) + + def test_generate_uri_path_parameter_with_documented_example(self): + uri_parameters = [{'Name': 'id', 'Required': True, 'ExampleValue': '1', 'Type': 'string', 'Format': None, 'Location': 'Path'}] + + RequestBuildHelper.generate_uri('/api/endpoint/{id}', uri_parameters) + + uri = s_render().decode('utf8', 'ignore') + self.assertEqual('/api/endpoint/1', uri) + + def test_generate_uri_single_query_parameter_with_documented_example(self): + uri_parameters = [{'Name': 'id', 'Required': True, 'ExampleValue': '1', 'Type': 'string', 'Format': None, 'Location': 'Query'}] + + RequestBuildHelper.generate_uri('/api/endpoint', uri_parameters) + + uri = s_render().decode('utf8', 'ignore') + self.assertEqual('/api/endpoint?id=1', uri) + + def test_generate_uri_single_query_parameter_with_multiple_documented_examples(self): + uri_parameters = [ + {'Name': 'id', 'Required': True, 'ExampleValue': '1', 'Type': 'string', 'Format': None, 'Location': 'Query'}, + {'Name': 'attr', 'Required': True, 'ExampleValue': '2', 'Type': 'string', 'Format': None, 'Location': 'Query'} + ] + + RequestBuildHelper.generate_uri('/api/endpoint', uri_parameters) + + uri = s_render().decode('utf8', 'ignore') + self.assertEqual('/api/endpoint?id=1&attr=2', uri) + + def test_generate_uri_combined_parameters(self): + ConfigurationManager.config = { + "fixed_url_attributes": { + "attr2": "20" + } + } + uri_parameters = [ + {'Name': 'id', 'Required': True, 'ExampleValue': '1', 'Type': 'string', 'Format': None, 'Location': 'Path'}, + {'Name': 'attr1', 'Required': True, 'ExampleValue': '2', 'Type': 'string', 'Format': None, 'Location': 'Query'}, + {'Name': 'attr2', 'Required': True, 'ExampleValue': '3', 'Type': 'integer', 'Format': 'int32', 'Location': 'Query'} + ] + + RequestBuildHelper.generate_uri('/api/endpoint/{id}', uri_parameters) + + uri = s_render().decode('utf8', 'ignore') + self.assertEqual('/api/endpoint/1?attr1=2&attr2=20', uri) diff --git a/parser/Parser.Tests/AttributeParserTests.cs b/parser/Parser.Tests/AttributeParserTests.cs index e14c68b..db1d454 100644 --- a/parser/Parser.Tests/AttributeParserTests.cs +++ b/parser/Parser.Tests/AttributeParserTests.cs @@ -19,6 +19,34 @@ namespace Parser.Tests Assert.IsNull(parsedAttribute); } + [Test] + public void ParsingPathAttributeWithPathLocation() + { + OpenApiParameter parameter = new OpenApiParameter + { + Schema = new OpenApiSchema { Type = "string", Format = null }, + In = ParameterLocation.Path + }; + + var parsedAttribute = AttributeParser.ParseAttribute(parameter); + + Assert.AreEqual("Path", parsedAttribute.Location); + } + + [Test] + public void ParsingPathAttributeWithQueryLocation() + { + OpenApiParameter parameter = new OpenApiParameter + { + Schema = new OpenApiSchema { Type = "string", Format = null }, + In = ParameterLocation.Query + }; + + var parsedAttribute = AttributeParser.ParseAttribute(parameter); + + Assert.AreEqual("Query", parsedAttribute.Location); + } + [Test] public void ParsingAttributeWithNoTypeOrFormatShouldReturnNull() { From c3469ff3f981e1c780b02e87ea8005e43971d321 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20St=C3=A1rek?= Date: Tue, 29 Oct 2019 16:11:09 +0100 Subject: [PATCH 4/4] Refactoring --- fuzzer/src/request_build_helper.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/fuzzer/src/request_build_helper.py b/fuzzer/src/request_build_helper.py index 3da60b2..876f673 100644 --- a/fuzzer/src/request_build_helper.py +++ b/fuzzer/src/request_build_helper.py @@ -8,7 +8,6 @@ from configuration_manager import ConfigurationManager class RequestBuildHelper(object): - # Content-length and Host are mandatory @staticmethod def generate_headers(config): @@ -36,12 +35,7 @@ class RequestBuildHelper(object): return headers is not None and header_name in headers @staticmethod - def generate_uri(uri, uri_parameters, fuzzable=False): - id_generator = _unique_uri_attribute_id() - - already_used_parameters: List[str] = [] - - # 1] Generate URI as it is in payloads file + def _generate_base_uri_path(uri, uri_parameters, id_generator, fuzzable, already_used_parameters): while True: try: # Find first not yet found parameter, if there is one @@ -62,15 +56,24 @@ class RequestBuildHelper(object): s_http_string(uri, fuzzable=False, encoding=EncodingTypes.ascii, name=name) break - # 2] Append another URI attributes + @staticmethod + def _generate_additional_query_parameters(uri_parameters, already_used_parameters, id_generator, fuzzable): for uri_parameter in uri_parameters: parameter_name = uri_parameter["Name"] if parameter_name not in already_used_parameters and uri_parameter["Location"] == "Query": prefix = "?" if "?" not in s_render().decode('ascii', 'ignore') else "&" - name = "URI attribute, default value: " + uri + ", id: " + next(id_generator) + name = "URI attribute, default value: " + parameter_name + ", id: " + next(id_generator) s_http_string(prefix + parameter_name + "=", fuzzable=False, encoding=EncodingTypes.ascii, name=name) RequestBuildHelper._append_parameter(parameter_name, id_generator, uri_parameters, fuzzable) + @staticmethod + def generate_uri(uri, uri_parameters, fuzzable=False): + id_generator = _unique_uri_attribute_id() + already_used_parameters: List[str] = [] + + RequestBuildHelper._generate_base_uri_path(uri, uri_parameters, id_generator, fuzzable, already_used_parameters) + RequestBuildHelper._generate_additional_query_parameters(uri_parameters, already_used_parameters, id_generator, fuzzable) + @staticmethod def _append_parameter(parameter_name, id_generator, uri_parameters, fuzzable): fixed_attributes = ConfigurationManager.config["fixed_url_attributes"] if "fixed_url_attributes" in ConfigurationManager.config else None