diff --git a/packages/google-api-core/google/api_core/path_template.py b/packages/google-api-core/google/api_core/path_template.py index 967299093d56..1a5662722155 100644 --- a/packages/google-api-core/google/api_core/path_template.py +++ b/packages/google-api-core/google/api_core/path_template.py @@ -28,6 +28,7 @@ import copy import functools import re +import urllib.parse from collections import deque # Regular expression for extracting variable parts from a path template. @@ -62,6 +63,190 @@ _MULTI_SEGMENT_PATTERN = r"(.+)" +class _PathValidationFailed(Exception): + """Internal exception used when an invalid path is encountered.""" + + +def _validate_multi_segment_value(val: str) -> bool: + """Validate a multi-segment wildcard value. + + This function implements the dot and double-dot traversal validation rule + for values matching '**'. It splits the value by '/' into segments and + simulates path traversal: + - '.' segments do not modify the segment count. + - '..' segments decrease the leftover segment count by 1. + - Any other segments increase the leftover segment count by 1. + + Validation fails if: + - Path traversal overflows to the left (leftover segment count drops below 0), + meaning it would traverse out of the multi-segment value boundaries. + - No segments remain after executing all path traversal commands (leftover + segment count is 0), meaning the entire value is consumed. + + Examples: + >>> _validate_multi_segment_value("instance/my-instance") + True + >>> _validate_multi_segment_value("instance/my-instance/..") + True + >>> _validate_multi_segment_value("instance/../..") + False + + Args: + val (str): The value matched to the '**' wildcard to validate. + + Returns: + bool: True if the value is valid and does not violate traversal boundaries, + False otherwise. + """ + if val in ("", ".", ".."): + return False + + segments = val.split("/") + leftover_segments = 0 + unseen_segments = len(segments) + + for segment in segments: + unseen_segments -= 1 + if segment == "..": + leftover_segments -= 1 + elif segment not in (".", ""): + leftover_segments += 1 + + if leftover_segments < 0: + return False + if leftover_segments > unseen_segments: + return True + + return leftover_segments > 0 + + +@functools.lru_cache(maxsize=1024) +def _build_capture_pattern(template_str: str) -> tuple[re.Pattern, tuple[str, ...]]: + """Build a regex pattern to capture wildcard matches from a template. + + This function parses a template string containing positional/named + wildcards ('*' or '**'), compiles it into a regular expression, and + records the order and types of all wildcards to allow extracting + sub-segments for individual validation. + + Args: + template_str (str): The template string (e.g. "projects/*/locations/*"). + + Returns: + tuple[re.Pattern, tuple[str, ...]]: A tuple containing: + - The compiled regex pattern string with capture groups. + - A list of wildcard type strings ('*' or '**') in matching order. + """ + wildcard_types = [] + parts = [] + last_idx = 0 + for match in _VARIABLE_RE.finditer(template_str): + literal = template_str[last_idx : match.start()] + parts.append(re.escape(literal)) + + positional = match.group("positional") + template = match.group("template") + + if positional == "*": + wildcard_types.append("*") + replaced = _SINGLE_SEGMENT_PATTERN + elif positional == "**": + wildcard_types.append("**") + replaced = _MULTI_SEGMENT_PATTERN + elif not template or template == "*": + wildcard_types.append("*") + replaced = _SINGLE_SEGMENT_PATTERN + elif template == "**": + wildcard_types.append("**") + replaced = _MULTI_SEGMENT_PATTERN + else: + sub_pattern, sub_types = _build_capture_pattern(template) + wildcard_types.extend(sub_types) + replaced = sub_pattern.pattern + + parts.append(replaced) + last_idx = match.end() + literal = template_str[last_idx:] + parts.append(re.escape(literal)) + + pattern = "".join(parts) + return re.compile(pattern), tuple(wildcard_types) + + +def _extract_and_validate_wildcards( + val: str, template_str: str | None, property_name: str | None = None +) -> None: + """Extract and validate wildcard variables against path traversal rules. + + This function attempts to structurally match the variable's value against + its template. If the value structurally matches, it extracts the substrings + corresponding to the individual wildcards and enforces safety constraints: + - Single-segment matches ('*') must not be exactly '.' or '..' because + this breaks the URI routing contract and leads to path traversal. + - Multi-segment matches ('**') are checked using _validate_multi_segment_value + to ensure path traversal commands do not consume the entire value or + escape the starting boundaries of the matched parameter. + + If the value does not structurally match the template, this function allows + it to pass through without error. This delegates the rejection of the malformed + string to the standard routing mechanisms (like `validate()`) to ensure that + `additional_bindings` are evaluated correctly. + + Examples: + >>> _extract_and_validate_wildcards("us-central1", None, "region") + None + >>> _extract_and_validate_wildcards("..", None, "region") + ValueError("Invalid value .. for region.") + >>> _extract_and_validate_wildcards( + ... "projects/my-proj/locations/.", "projects/*/locations/*", "parent" + ... ) + ValueError("Invalid value projects/my-proj/locations/. for parent.") + + Args: + val (str): The raw string value to validate. + template_str (str): The template string of the variable (e.g. 'projects/*/locations/*'). + property_name (str | None): The name of the property being validated, used + to construct descriptive error messages. + + Raises: + ValueError: If a wildcard within a structurally valid value violates path traversal rules. + """ + try: + if template_str is None or template_str == "*": + # Single-segment templates (None or "*") cannot match exactly "." or ".." + # and cannot have multi-segment paths resolving to 0 segments. + # + # Empty strings pose no traversal security risk here. + if val and not _validate_multi_segment_value(val): + raise _PathValidationFailed() + elif template_str == "**": + # Multi-segment templates ("**") must represent at least one valid, + # non-escaped segment. + if not _validate_multi_segment_value(val): + raise _PathValidationFailed() + else: + # Compile the sub-template into a regex capture pattern + # to isolate and validate individual wildcard values. + pattern, wildcard_types = _build_capture_pattern(template_str) + + m = pattern.fullmatch(val) + if m is not None: + # Validate each wildcard value within its matched boundaries, + # preventing traversals from escaping their structural positions. + for captured_val in m.groups(): + if not _validate_multi_segment_value(captured_val): + raise _PathValidationFailed() + else: + # For values that don't match the pattern, ensure the value doesn't + # resolve to 0 segments (e.g. "projects/.."). + if val and not _validate_multi_segment_value(val): + raise _PathValidationFailed() + except _PathValidationFailed: + raise ValueError( + f"Invalid value {val} for {property_name or 'positional variable'}." + ) + + def _expand_variable_match(positional_vars, named_vars, match): """Expand a matched variable with its value. @@ -81,9 +266,13 @@ def _expand_variable_match(positional_vars, named_vars, match): """ positional = match.group("positional") name = match.group("name") + template = match.group("template") + if name is not None: try: - return str(named_vars[name]) + val = str(named_vars[name]) + _extract_and_validate_wildcards(val, template, name) + return urllib.parse.quote(val, safe="/") except KeyError: raise ValueError( "Named variable '{}' not specified and needed by template " @@ -91,7 +280,9 @@ def _expand_variable_match(positional_vars, named_vars, match): ) elif positional is not None: try: - return str(positional_vars.pop(0)) + val = str(positional_vars.pop(0)) + _extract_and_validate_wildcards(val, positional, None) + return urllib.parse.quote(val, safe="/") except IndexError: raise ValueError( "Positional variable not specified and needed by template " diff --git a/packages/google-api-core/tests/unit/test_path_template.py b/packages/google-api-core/tests/unit/test_path_template.py index e621d5151f58..cb9ffe532a74 100644 --- a/packages/google-api-core/tests/unit/test_path_template.py +++ b/packages/google-api-core/tests/unit/test_path_template.py @@ -64,6 +64,29 @@ {"name": "parent/child/object"}, "/v1/a/parent/child/object", ], + # Encoding / Metacharacters in positional and named params + ["/v1/*", ["..?$httpMethod=DELETE#"], {}, "/v1/..%3F%24httpMethod%3DDELETE%23"], + ["/v1/**", ["path/../with/?and#"], {}, "/v1/path/../with/%3Fand%23"], + [ + "/v1/{name}", + [], + {"name": "..?$httpMethod=DELETE#"}, + "/v1/..%3F%24httpMethod%3DDELETE%23", + ], + [ + "/v1/{name=**}", + [], + {"name": "path/../with/?and#"}, + "/v1/path/../with/%3Fand%23", + ], + [ + "/v3/{session=projects/*/locations/*/agents/*/sessions/*}:detectIntent", + [], + { + "session": "projects/cx/locations/global/agents/a1/sessions/..?$httpMethod=DELETE#" + }, + "/v3/projects/cx/locations/global/agents/a1/sessions/..%3F%24httpMethod%3DDELETE%23:detectIntent", + ], ], ) def test_expand_success(tmpl, args, kwargs, expected_result): @@ -651,3 +674,223 @@ def helper_test_transcode(http_options_list, expected_result_list): if expected_result_list[2]: expected_result["body"] = expected_result_list[2] return (http_options, expected_result) + + +@pytest.mark.parametrize( + "tmpl, args, kwargs, expected_err_match", + [ + # Positional * with . or .. + [ + "/v1/*", + ["."], + {}, + "Invalid value \\. for positional variable\\.", + ], + [ + "/v1/*", + [".."], + {}, + "Invalid value \\.\\. for positional variable\\.", + ], + # Named matching * with . or .. + [ + "/compute/v1/projects/{project}/regions/{region}/addresses", + [], + {"project": "my-project", "region": ".."}, + "Invalid value \\.\\. for region\\.", + ], + [ + "/compute/v1/projects/{project}/regions/{region}/addresses", + [], + {"project": "my-project", "region": "."}, + "Invalid value \\. for region\\.", + ], + # Sub-template named matching * with . or .. + [ + "/v2/{parent=projects/*/locations/*}/content:inspect", + [], + {"parent": "projects/my-project/locations/.."}, + "Invalid value projects/my-project/locations/\\.\\. for parent\\.", + ], + [ + "/v2/{parent=projects/*/locations/*}/content:inspect", + [], + {"parent": "projects/my-project/locations/."}, + "Invalid value projects/my-project/locations/\\. for parent\\.", + ], + # Non-matching values with path traversal (bypass prevention) + [ + "/v2/{parent=projects/*/locations/*}/content:inspect", + [], + {"parent": "projects/.."}, + "Invalid value projects/\\.\\. for parent\\.", + ], + [ + "/v1/{name}", + [], + {"name": "projects/.."}, + "Invalid value projects/\\.\\. for name\\.", + ], + ], +) +def test_path_traversal_dots_validation_star(tmpl, args, kwargs, expected_err_match): + with pytest.raises(ValueError, match=expected_err_match): + path_template.expand(tmpl, *args, **kwargs) + + +@pytest.mark.parametrize( + "name_val, expected_path", + [ + ( + "projects/my-project/monitoredResourceDescriptors/instance/my-instance/..", + "/v3/projects/my-project/monitoredResourceDescriptors/instance/my-instance/..", + ), + ( + "projects/my-project/monitoredResourceDescriptors/instance/my-instance/.", + "/v3/projects/my-project/monitoredResourceDescriptors/instance/my-instance/.", + ), + ( + "projects/my-project/monitoredResourceDescriptors/a/b/c/d/e/../../../..", + "/v3/projects/my-project/monitoredResourceDescriptors/a/b/c/d/e/../../../..", + ), + ], +) +def test_path_traversal_dots_validation_double_star_valid(name_val, expected_path): + assert ( + path_template.expand( + "/v3/{name=projects/*/monitoredResourceDescriptors/**}", + name=name_val, + ) + == expected_path + ) + + +@pytest.mark.parametrize( + "name_val", + [ + "projects/my-project/monitoredResourceDescriptors/.", + "projects/my-project/monitoredResourceDescriptors/instance/my-instance/../..", + "projects/my-project/monitoredResourceDescriptors/instance/../my-instance/..", + "projects/my-project/monitoredResourceDescriptors/..", + "projects/my-project/monitoredResourceDescriptors/instance/../..", + "projects/my-project/monitoredResourceDescriptors/a/b/../../../c/d/e/..", + "projects/my-project/monitoredResourceDescriptors/instance//..", + ], +) +def test_path_traversal_dots_validation_double_star_invalid(name_val): + with pytest.raises(ValueError, match=r"Invalid value .* for name\."): + path_template.expand( + "/v3/{name=projects/*/monitoredResourceDescriptors/**}", + name=name_val, + ) + + +@pytest.mark.parametrize( + "tmpl, kwargs, expected_result", + [ + ["/v1/{name}", {"name": "abc-._~"}, "/v1/abc-._~"], + ["/v1/{name=**}", {"name": "abc-._~/"}, "/v1/abc-._~/"], + ["/v1/{name}", {"name": "a/b"}, "/v1/a/b"], + ["/v1/{name}", {"name": "a$b?c=d#e f"}, "/v1/a%24b%3Fc%3Dd%23e%20f"], + ], +) +def test_percent_encoding_unreserved_characters(tmpl, kwargs, expected_result): + result = path_template.expand(tmpl, **kwargs) + assert result == expected_result + # For single-segment with '/', validate should fail because '/' is preserved + if "/" in kwargs.get("name", "") and tmpl == "/v1/{name}": + assert not path_template.validate(tmpl, result) + + +@pytest.mark.parametrize( + "tmpl, args, kwargs, expected_err_match", + [ + ("/v1/**", [".."], {}, r"Invalid value .* for positional variable\."), + ("/v1/{name=**}", [], {"name": ".."}, r"Invalid value .* for name\."), + ], +) +def test_path_traversal_dots_validation_bare_double_star( + tmpl, args, kwargs, expected_err_match +): + with pytest.raises(ValueError, match=expected_err_match): + path_template.expand(tmpl, *args, **kwargs) + + +@pytest.mark.parametrize( + "template_str, expected_pattern, expected_wildcards", + [ + ( + "projects/{project}/locations/{location}", + "projects/([^/]+)/locations/([^/]+)", + ("*", "*"), + ), + ("projects/{project=**}", "projects/(.+)", ("**",)), + ("projects/{project=locations/*}", "projects/locations/([^/]+)", ("*",)), + ("projects/*/locations/**", "projects/([^/]+)/locations/(.+)", ("*", "**")), + ("projects/abc", "projects/abc", ()), + ("projects/my-project", r"projects/my\-project", ()), + ( + r"my-test/{id}/v1.0?abc+def-g|h()\\", + r"my\-test/([^/]+)/v1\.0\?abc\+def\-g\|h\(\)\\\\", + ("*",), + ), + ], +) +def test_build_capture_pattern(template_str, expected_pattern, expected_wildcards): + pattern, wildcards = path_template._build_capture_pattern(template_str) + assert pattern.pattern == expected_pattern + assert wildcards == expected_wildcards + + +@pytest.mark.parametrize( + "val, expected_valid", + [ + ("a/b/c", True), + ("a/../b", True), + ("a/b/c/d/e/../../../..", True), + ("a/b/../..", False), + ("a/b/../../..", False), + ("", False), + (".", False), + ("..", False), + ("", False), + ("/", False), + ("//", False), + ("a/../../b", False), + ("../..", False), + ("instance//..", False), + ("instance/my-instance///../..", False), + ], +) +def test_validate_multi_segment_value(val, expected_valid): + result = path_template._validate_multi_segment_value(val) + assert result == expected_valid + + +@pytest.mark.parametrize( + "val, template_str, expect_error", + [ + ("api", None, False), + ("api", "*", False), + (".", None, True), + ("..", None, True), + (".", "*", True), + ("..", "*", True), + ("", "*", False), + ("api/v1", "**", False), + (".", "**", True), + ("..", "**", True), + ("", "**", True), + ("projects/my-proj/locations/us-central1", "projects/*/locations/*", False), + ("projects/my-proj/locations/.", "projects/*/locations/*", True), + ("projects/my-proj/locations/..", "projects/*/locations/*", True), + ("projects/../locations/us-central1", "projects/*/locations/*", True), + ], +) +def test_extract_and_validate_wildcards(val, template_str, expect_error): + if expect_error: + with pytest.raises(ValueError): + path_template._extract_and_validate_wildcards(val, template_str) + else: + # Should not raise any exception + path_template._extract_and_validate_wildcards(val, template_str)