diff --git a/crt/aws-crt-cpp b/crt/aws-crt-cpp index 72f84bc32746..0463563f9f65 160000 --- a/crt/aws-crt-cpp +++ b/crt/aws-crt-cpp @@ -1 +1 @@ -Subproject commit 72f84bc327462f405c4994228fffe1eeb16cca72 +Subproject commit 0463563f9f656a493ec22ca962c2464bc8b831ab diff --git a/prefetch_crt_dependency.sh b/prefetch_crt_dependency.sh index ba0d4b35cdcb..4fea59687943 100755 --- a/prefetch_crt_dependency.sh +++ b/prefetch_crt_dependency.sh @@ -3,21 +3,21 @@ # SPDX-License-Identifier: Apache-2.0. CRT_URI_PREFIX=https://codeload.github.com/awslabs -CRT_URI=${CRT_URI_PREFIX}/aws-crt-cpp/zip/72f84bc327462f405c4994228fffe1eeb16cca72 # v0.42.2 +CRT_URI=${CRT_URI_PREFIX}/aws-crt-cpp/zip/0463563f9f656a493ec22ca962c2464bc8b831ab # v0.43.2 AWS_C_AUTH_URI=${CRT_URI_PREFIX}/aws-c-auth/zip/4b5d524bf1a511b05e0fffe5bdc51800770b9427 # v0.10.4 -AWS_C_CAL_URI=${CRT_URI_PREFIX}/aws-c-cal/zip/9edd8eac2b21ca6a04535b91d60d361c2f1bb60f # v0.9.14 +AWS_C_CAL_URI=${CRT_URI_PREFIX}/aws-c-cal/zip/8aa2a48a09f93c65d4cf06388e143a6584de6321 # v0.9.15 AWS_C_COMMON_URI=${CRT_URI_PREFIX}/aws-c-common/zip/3c69b871dfa1815231802febf1bb6899f84cccdb # v0.14.3 AWS_C_COMPRESSION_URI=${CRT_URI_PREFIX}/aws-c-compression/zip/d8264e64f698341eb03039b96b4f44702a9b3f83 # v0.3.2 AWS_C_EVENT_STREAM_URI=${CRT_URI_PREFIX}/aws-c-event-stream/zip/51bef3c44e1058b1689751539170b2e0f589ccdb # v0.7.1 AWS_C_HTTP_URI=${CRT_URI_PREFIX}/aws-c-http/zip/8aefd899fc3210bfd0e3fd414011a3cb708bf6e4 # v0.11.0 -AWS_C_IO_URI=${CRT_URI_PREFIX}/aws-c-io/zip/54350963b64dfc6c4b0ea623b08aa252aae3d7d7 # v0.27.4 +AWS_C_IO_URI=${CRT_URI_PREFIX}/aws-c-io/zip/e2946c99521fa12d285c9a0829c92b1bf713922b # v0.27.5 AWS_C_MQTT_URI=${CRT_URI_PREFIX}/aws-c-mqtt/zip/2ef9605ec9c50bea3f921e08022ddd57eed70901 # v0.16.0 -AWS_C_S3_URI=${CRT_URI_PREFIX}/aws-c-s3/zip/1f29ef8871a27dc8b90325418780659bac534d71 # v0.13.1 -AWS_C_SDKUTILS_URI=${CRT_URI_PREFIX}/aws-c-sdkutils/zip/cb14fea362c82c995eebd34e2e96590ab4e0ed58 # v0.2.7 +AWS_C_S3_URI=${CRT_URI_PREFIX}/aws-c-s3/zip/a852faa2df3ab2b31fb4cfd64fd3379a2f4ae22e # v0.13.2 +AWS_C_SDKUTILS_URI=${CRT_URI_PREFIX}/aws-c-sdkutils/zip/528b9dfff4a804b334875ecf8a0471f7d1366f24 # v0.2.8 AWS_CHECKSUMS_URI=${CRT_URI_PREFIX}/aws-checksums/zip/1d5f2f1f3e5d013aae8810878ceb5b3f6f258c4e # v0.2.10 -AWS_LC_URI=${CRT_URI_PREFIX}/aws-lc/zip/683ebde4bf3bcc016a9a710ad6b49c0c91b59161 # v5.2.0 -S2N_URI=${CRT_URI_PREFIX}/s2n/zip/f5f6c6c2ce2370de1aa3ade6899a7321d1127bb8 # v1.7.5 +AWS_LC_URI=${CRT_URI_PREFIX}/aws-lc/zip/f6acf748df0ea6157d55e640730b38d21a7751cd # v5.4.0 +S2N_URI=${CRT_URI_PREFIX}/s2n/zip/66b1c94d1dfc99b237427cbde230eca63bb8b89c # v1.7.6 echo "Removing CRT" diff --git a/src/aws-cpp-sdk-core/include/aws/core/utils/base64/Base64.h b/src/aws-cpp-sdk-core/include/aws/core/utils/base64/Base64.h index 503e73a24cfb..c231b12eea1e 100644 --- a/src/aws-cpp-sdk-core/include/aws/core/utils/base64/Base64.h +++ b/src/aws-cpp-sdk-core/include/aws/core/utils/base64/Base64.h @@ -57,11 +57,6 @@ namespace Aws * Calculates the length of an encoded base64 string based on the buffer being encoded */ static size_t CalculateBase64EncodedLength(const ByteBuffer& buffer); - - private: - char m_mimeBase64EncodingTable[64]; - uint8_t m_mimeBase64DecodingTable[256]; - }; } // namespace Base64 diff --git a/src/aws-cpp-sdk-core/source/utils/base64/Base64.cpp b/src/aws-cpp-sdk-core/source/utils/base64/Base64.cpp index 2103d6d5a687..c2ce37674698 100644 --- a/src/aws-cpp-sdk-core/source/utils/base64/Base64.cpp +++ b/src/aws-cpp-sdk-core/source/utils/base64/Base64.cpp @@ -3,146 +3,46 @@ * SPDX-License-Identifier: Apache-2.0. */ +#include #include -#include +#include using namespace Aws::Utils::Base64; -static const uint8_t SENTINEL_VALUE = 255; -static const char BASE64_ENCODING_TABLE_MIME[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; +namespace Aws { +namespace Utils { +namespace Base64 { -namespace Aws -{ -namespace Utils -{ -namespace Base64 -{ - -Base64::Base64(const char *encodingTable) -{ - if(encodingTable == nullptr) - { - encodingTable = BASE64_ENCODING_TABLE_MIME; - } - - size_t encodingTableLength = strlen(encodingTable); - if(encodingTableLength != 64) - { - encodingTable = BASE64_ENCODING_TABLE_MIME; - encodingTableLength = 64; - } - - memcpy(m_mimeBase64EncodingTable, encodingTable, encodingTableLength); - - memset((void *)m_mimeBase64DecodingTable, 0, 256); - - for(uint32_t i = 0; i < encodingTableLength; ++i) - { - uint32_t index = static_cast(m_mimeBase64EncodingTable[i]); - m_mimeBase64DecodingTable[index] = static_cast(i); - } - - m_mimeBase64DecodingTable[(uint32_t)'='] = SENTINEL_VALUE; +namespace { +Aws::Crt::ByteCursor AsCursor(const Aws::Utils::ByteBuffer& buffer) { + return Aws::Crt::ByteCursorFromArray(buffer.GetUnderlyingData(), buffer.GetLength()); } -Aws::String Base64::Encode(const Aws::Utils::ByteBuffer& buffer) const -{ - size_t bufferLength = buffer.GetLength(); - size_t blockCount = (bufferLength + 2) / 3; - size_t remainderCount = (bufferLength % 3); - - Aws::String outputString; - outputString.reserve(CalculateBase64EncodedLength(buffer)); - - for(size_t i = 0; i < bufferLength; i += 3 ) - { - uint32_t block = buffer[ i ]; - - block <<= 8; - if (i + 1 < bufferLength) - { - block = block | buffer[ i + 1 ]; - } - - block <<= 8; - if (i + 2 < bufferLength) - { - block = block | buffer[ i + 2 ]; - } - - outputString.push_back(m_mimeBase64EncodingTable[(block >> 18) & 0x3F]); - outputString.push_back(m_mimeBase64EncodingTable[(block >> 12) & 0x3F]); - outputString.push_back(m_mimeBase64EncodingTable[(block >> 6) & 0x3F]); - outputString.push_back(m_mimeBase64EncodingTable[block & 0x3F]); - } - - if(remainderCount > 0) - { - outputString[blockCount * 4 - 1] = '='; - if(remainderCount == 1) - { - outputString[blockCount * 4 - 2] = '='; - } - } - - return outputString; +Aws::Crt::ByteCursor AsCursor(const Aws::String& str) { + return Aws::Crt::ByteCursorFromArray(reinterpret_cast(str.data()), str.length()); } +} // namespace -Aws::Utils::ByteBuffer Base64::Decode(const Aws::String& str) const -{ - size_t decodedLength = CalculateBase64DecodedLength(str); - - Aws::Utils::ByteBuffer buffer(decodedLength); - - const char* rawString = str.c_str(); - size_t blockCount = str.length() / 4; - for(size_t i = 0; i < blockCount; ++i) - { - size_t stringIndex = i * 4; +Base64::Base64(const char* encodingTable) { AWS_UNREFERENCED_PARAM(encodingTable); } - uint32_t value1 = m_mimeBase64DecodingTable[uint32_t(rawString[stringIndex])]; - uint32_t value2 = m_mimeBase64DecodingTable[uint32_t(rawString[++stringIndex])]; - uint32_t value3 = m_mimeBase64DecodingTable[uint32_t(rawString[++stringIndex])]; - uint32_t value4 = m_mimeBase64DecodingTable[uint32_t(rawString[++stringIndex])]; - - size_t bufferIndex = i * 3; - buffer[bufferIndex] = static_cast((value1 << 2) | ((value2 >> 4) & 0x03)); - if(value3 != SENTINEL_VALUE) - { - buffer[++bufferIndex] = static_cast(((value2 << 4) & 0xF0) | ((value3 >> 2) & 0x0F)); - if(value4 != SENTINEL_VALUE) - { - buffer[++bufferIndex] = static_cast((value3 & 0x03) << 6 | value4); - } - } - } - - return buffer; +Aws::String Base64::Encode(const Aws::Utils::ByteBuffer& buffer) const { + const auto encoded = Aws::Crt::Base64Encode(AsCursor(buffer)); + return {encoded.data(), encoded.size()}; } -size_t Base64::CalculateBase64DecodedLength(const Aws::String& b64input) -{ - const size_t len = b64input.length(); - if(len < 2) - { - return 0; - } - - size_t padding = 0; - - if (b64input[len - 1] == '=' && b64input[len - 2] == '=') //last two chars are = - padding = 2; - else if (b64input[len - 1] == '=') //last char is = - padding = 1; +Aws::Utils::ByteBuffer Base64::Decode(const Aws::String& str) const { + const auto decoded = Aws::Crt::Base64Decode(AsCursor(str)); + return {decoded.data(), decoded.size()}; +} - return (len * 3 / 4 - padding); +size_t Base64::CalculateBase64EncodedLength(const Aws::Utils::ByteBuffer& buffer) { + return Aws::Crt::Base64EncodedLength(AsCursor(buffer)); } -size_t Base64::CalculateBase64EncodedLength(const Aws::Utils::ByteBuffer& buffer) -{ - return 4 * ((buffer.GetLength() + 2) / 3); +size_t Base64::CalculateBase64DecodedLength(const Aws::String& b64input) { + return Aws::Crt::Base64DecodedLength(AsCursor(b64input)); } -} // namespace Base64 -} // namespace Utils -} // namespace Aws \ No newline at end of file +} // namespace Base64 +} // namespace Utils +} // namespace Aws diff --git a/tests/aws-cpp-sdk-core-tests/utils/HashingUtilsTest.cpp b/tests/aws-cpp-sdk-core-tests/utils/HashingUtilsTest.cpp index d32f711e8301..d442cca014c7 100644 --- a/tests/aws-cpp-sdk-core-tests/utils/HashingUtilsTest.cpp +++ b/tests/aws-cpp-sdk-core-tests/utils/HashingUtilsTest.cpp @@ -6,6 +6,7 @@ #include #include +#include #include @@ -68,6 +69,24 @@ TEST_F(HashingUtilsTest, TestBase64Decoding) ASSERT_EQ(ByteBuffer((unsigned char*) "foobar", 6), test7); } +TEST_F(HashingUtilsTest, TestBase64DecodeNeverWritesMoreThanCalculatedLength) +{ + Aws::Vector inputs = {"", "X", "Zg==", "Zm8=", "Zm9v", "Zm9vYg==", "Zm9vYmE=", "Zm9vYmFy", + "AAAA=", "AAAAA=", "AAAAAA=", "AAAAAAA=", "AB=D", "=", "==", "===", "====", + "A===", "AA==", "AAA=", "//++", "AAAA"}; + for (int byte = 0x80; byte <= 0xFF; ++byte) + { + inputs.emplace_back(Aws::String{static_cast(byte)} + "AAA"); + } + + for (const auto& input : inputs) + { + ASSERT_LE(HashingUtils::Base64Decode(input).GetLength(), + Aws::Utils::Base64::Base64::CalculateBase64DecodedLength(input)) + << "input: " << input; + } +} + TEST_F(HashingUtilsTest, TestHexEncodingDecoding) { unsigned char beforeHexEncoding[32] = diff --git a/tools/code-generation/generator/src/main/resources/com/amazonaws/util/awsclientgenerator/velocity/cpp/endpoint/EndpointProviderHeader.vm b/tools/code-generation/generator/src/main/resources/com/amazonaws/util/awsclientgenerator/velocity/cpp/endpoint/EndpointProviderHeader.vm index 5c0f530794e1..92eb290f1663 100644 --- a/tools/code-generation/generator/src/main/resources/com/amazonaws/util/awsclientgenerator/velocity/cpp/endpoint/EndpointProviderHeader.vm +++ b/tools/code-generation/generator/src/main/resources/com/amazonaws/util/awsclientgenerator/velocity/cpp/endpoint/EndpointProviderHeader.vm @@ -8,6 +8,11 @@ #set($epBuiltInClassName = "${metadata.classNamePrefix}BuiltInParameters") #set($exportMacro = "${CppViewHelper.computeExportValue($metadata.classNamePrefix)}") #set($externMacro = "AWS_${metadata.classNamePrefix.toUpperCase()}_EXTERN") +#if($serviceModel.skipEndpointRulesBlob) +#set($epProviderType = "BDDEndpointProvider") +#else +#set($epProviderType = "DefaultEndpointProvider") +#end #pragma once \#include #if($serviceModel.hasServiceSpecificClientConfig()) @@ -15,7 +20,7 @@ #else \#include #end -\#include +\#include \#include \#include \#include @@ -33,7 +38,7 @@ using ${metadata.classNamePrefix}ClientConfiguration = Aws::${serviceNamespace}: #end using EndpointParameters = Aws::Endpoint::EndpointParameters; using Aws::Endpoint::EndpointProviderBase; -using Aws::Endpoint::DefaultEndpointProvider; +using Aws::Endpoint::${epProviderType}; #if ($serviceModel.endpointRules) #if ($serviceModel.clientContextParams) @@ -92,7 +97,7 @@ using ${metadata.classNamePrefix}EndpointProviderBase = EndpointProviderBase<${metadata.classNamePrefix}ClientConfiguration, ${epBuiltInClassName}, ${epContextClassName}>; using ${metadata.classNamePrefix}DefaultEpProviderBase = - DefaultEndpointProvider<${metadata.classNamePrefix}ClientConfiguration, ${epBuiltInClassName}, ${epContextClassName}>; + ${epProviderType}<${metadata.classNamePrefix}ClientConfiguration, ${epBuiltInClassName}, ${epContextClassName}>; #if($serviceModel.hasServiceSpecificClientConfig() || $serviceModel.clientContextParams) } // namespace Endpoint @@ -107,7 +112,7 @@ ${externMacro} template class ${exportMacro} Aws::Endpoint::EndpointProviderBase<${serviceNamespace}::Endpoint::${metadata.classNamePrefix}ClientConfiguration, ${serviceNamespace}::Endpoint::${epBuiltInClassName}, ${serviceNamespace}::Endpoint::${epContextClassName}>; ${externMacro} template class ${exportMacro} - Aws::Endpoint::DefaultEndpointProvider<${serviceNamespace}::Endpoint::${metadata.classNamePrefix}ClientConfiguration, ${serviceNamespace}::Endpoint::${epBuiltInClassName}, ${serviceNamespace}::Endpoint::${epContextClassName}>; + Aws::Endpoint::${epProviderType}<${serviceNamespace}::Endpoint::${metadata.classNamePrefix}ClientConfiguration, ${serviceNamespace}::Endpoint::${epBuiltInClassName}, ${serviceNamespace}::Endpoint::${epContextClassName}>; } // namespace Endpoint namespace ${serviceNamespace} diff --git a/tools/code-generation/generator/src/main/resources/com/amazonaws/util/awsclientgenerator/velocity/cpp/endpoint/EndpointProviderSource.vm b/tools/code-generation/generator/src/main/resources/com/amazonaws/util/awsclientgenerator/velocity/cpp/endpoint/EndpointProviderSource.vm index 0049c13d2010..da352420e8f4 100644 --- a/tools/code-generation/generator/src/main/resources/com/amazonaws/util/awsclientgenerator/velocity/cpp/endpoint/EndpointProviderSource.vm +++ b/tools/code-generation/generator/src/main/resources/com/amazonaws/util/awsclientgenerator/velocity/cpp/endpoint/EndpointProviderSource.vm @@ -6,6 +6,11 @@ #set($endpointPrefix = $metadata.endpointPrefix) #set($epContextClassName = "${metadata.classNamePrefix}ClientContextParameters") #set($epBuiltInClassName = "${metadata.classNamePrefix}BuiltInParameters") +#if($serviceModel.skipEndpointRulesBlob) +#set($epProviderType = "BDDEndpointProvider") +#else +#set($epProviderType = "DefaultEndpointProvider") +#end \#include #if ($serviceModel.endpointRules) \#include @@ -24,7 +29,7 @@ template class Aws::Endpoint::EndpointProviderBase<${serviceNamespace}::Endpoint ${serviceNamespace}::Endpoint::${epBuiltInClassName}, ${serviceNamespace}::Endpoint::${epContextClassName}>; -template class Aws::Endpoint::DefaultEndpointProvider<${serviceNamespace}::Endpoint::${metadata.classNamePrefix}ClientConfiguration, +template class Aws::Endpoint::${epProviderType}<${serviceNamespace}::Endpoint::${metadata.classNamePrefix}ClientConfiguration, ${serviceNamespace}::Endpoint::${epBuiltInClassName}, ${serviceNamespace}::Endpoint::${epContextClassName}>; } // namespace Endpoint diff --git a/tools/code-generation/smithy/cpp-codegen/build.gradle.kts b/tools/code-generation/smithy/cpp-codegen/build.gradle.kts index 3cf6aa1a7281..27f4212fd3e4 100644 --- a/tools/code-generation/smithy/cpp-codegen/build.gradle.kts +++ b/tools/code-generation/smithy/cpp-codegen/build.gradle.kts @@ -45,6 +45,9 @@ tasks.register("generate-smithy-build") { val c2jMapStr: String = project.findProperty("c2jMap")?.toString() ?: "{}" val namespaceMappings: String = project.findProperty("namespaceMappings")?.toString() ?: "{}" val generateModels: Boolean = project.findProperty("generateModels")?.toString()?.toBoolean() ?: false + val generateEndpointRules: Boolean = project.findProperty("generateEndpointRules")?.toString()?.toBoolean() ?: false + val bddBytecoderPath: String = project.findProperty("bddBytecoderPath")?.toString() ?: "" + val pythonExecutable: String = project.findProperty("pythonExecutable")?.toString() ?: "python3" fileTree(models).filter { it.isFile }.files.forEach eachFile@{ file -> val model = Model.assembler() @@ -74,6 +77,14 @@ tasks.register("generate-smithy-build") { .withMember("namespaceMappings", Node.from(namespaceMappings)) .build()) } + if (generateEndpointRules) { + pluginsNode = pluginsNode.withMember("smithy-cpp-codegen-endpoint-rules", Node.objectNodeBuilder() + .withMember("c2jMap", Node.from(c2jMapStr)) + .withMember("namespaceMappings", Node.from(namespaceMappings)) + .withMember("bddBytecoderPath", Node.from(bddBytecoderPath)) + .withMember("pythonExecutable", Node.from(pythonExecutable)) + .build()) + } val projectionContents = Node.objectNodeBuilder() .withMember("imports", Node.fromStrings("${models.absolutePath}${File.separator}${file.name}")) @@ -127,6 +138,14 @@ tasks.register("generate-smithy-build") { .withMember("namespaceMappings", Node.from(namespaceMappings)) .build()) } + if (generateEndpointRules) { + s3CrtPluginsNode = s3CrtPluginsNode.withMember("smithy-cpp-codegen-endpoint-rules", Node.objectNodeBuilder() + .withMember("c2jMap", Node.from(c2jMapStr)) + .withMember("namespaceMappings", Node.from(namespaceMappings)) + .withMember("bddBytecoderPath", Node.from(bddBytecoderPath)) + .withMember("pythonExecutable", Node.from(pythonExecutable)) + .build()) + } val s3CrtProjectionContents = Node.objectNodeBuilder() .withMember("imports", Node.fromStrings(s3ModelFile.absolutePath)) .withMember("plugins", s3CrtPluginsNode) diff --git a/tools/code-generation/smithy/cpp-codegen/smithy-cpp-codegen/src/main/java/com/amazonaws/util/awsclientsmithygenerator/generators/ServiceNameUtil.java b/tools/code-generation/smithy/cpp-codegen/smithy-cpp-codegen/src/main/java/com/amazonaws/util/awsclientsmithygenerator/generators/ServiceNameUtil.java index 65dc089598ec..a47c2a320c3d 100644 --- a/tools/code-generation/smithy/cpp-codegen/smithy-cpp-codegen/src/main/java/com/amazonaws/util/awsclientsmithygenerator/generators/ServiceNameUtil.java +++ b/tools/code-generation/smithy/cpp-codegen/smithy-cpp-codegen/src/main/java/com/amazonaws/util/awsclientsmithygenerator/generators/ServiceNameUtil.java @@ -132,7 +132,7 @@ public static String capitalize(String str) { } // Match C2jModelToGeneratorModelTransformer.sanitizeServiceAbbreviation() exactly - private static String sanitizeServiceAbbreviation(String serviceAbbreviation) { + public static String sanitizeServiceAbbreviation(String serviceAbbreviation) { return serviceAbbreviation.replace(" ", "").replace("-", "").replace("_", "").replace("Amazon", "").replace("AWS", "").replace("/", ""); } @@ -165,6 +165,19 @@ public static String getExportMacro(ServiceShape service, Map se return "AWS_" + serviceName.toUpperCase() + "_API"; } + /** + * Returns the hidden-visibility macro for a service (e.g., "AWS_KINESIS_LOCAL"). + * Follows C2J convention: AWS_{UPPERCASED_SERVICE_NAME}_LOCAL + * + * @param service The service shape to generate the macro for + * @param serviceMap Service ID mappings for namespace overrides (reserved for future consistency with getSmithyServiceName) + * @return The local macro in format AWS_{SERVICE_NAME}_LOCAL + */ + public static String getLocalMacro(ServiceShape service, Map serviceMap) { + String serviceName = getServiceName(service); + return "AWS_" + serviceName.toUpperCase() + "_LOCAL"; + } + public static boolean isS3CrtProjection(ServiceShape service) { String serviceId = service.getTrait(ServiceTrait.class) .map(ServiceTrait::getSdkId) diff --git a/tools/code-generation/smithy/cpp-codegen/smithy-cpp-codegen/src/main/java/com/amazonaws/util/awsclientsmithygenerator/generators/endpointrules/BddBytecoder.java b/tools/code-generation/smithy/cpp-codegen/smithy-cpp-codegen/src/main/java/com/amazonaws/util/awsclientsmithygenerator/generators/endpointrules/BddBytecoder.java new file mode 100644 index 000000000000..989eb8ce4998 --- /dev/null +++ b/tools/code-generation/smithy/cpp-codegen/smithy-cpp-codegen/src/main/java/com/amazonaws/util/awsclientsmithygenerator/generators/endpointrules/BddBytecoder.java @@ -0,0 +1,69 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ +package com.amazonaws.util.awsclientsmithygenerator.generators.endpointrules; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; + +/** + * Shells out to the in-tree {@code bdd-bytecoder.py} compiler to turn the JSON form of a + * {@code smithy.rules#endpointBdd} trait into the binary BDD bytecode blob consumed by the + * CRT {@code BddEngine}. The bytecoder path and python executable are supplied by the caller + * (threaded in from {@code smithy_cpp_gen.py} as gradle properties) because the gradle working + * directory cannot reach the compiler under {@code crt/} on its own. + */ +public final class BddBytecoder { + + private BddBytecoder() {} + + /** + * @param pythonExecutable python interpreter to run (e.g. "python3"). + * @param bytecoderPath absolute path to bdd-bytecoder.py. + * @param traitJson the endpointBdd trait serialized as JSON (the compiler's input). + * @param serviceLabel used only to label temp files / error messages. + * @return the compiled binary bytecode. + */ + public static byte[] compile(String pythonExecutable, String bytecoderPath, String traitJson, String serviceLabel) { + String tempPrefix = "bdd-" + serviceLabel.replaceAll("[^a-zA-Z0-9._-]", "_") + "-"; + try (TempFile input = new TempFile(tempPrefix, ".json"); + TempFile output = new TempFile(tempPrefix, ".bin")) { + Files.writeString(input.path, traitJson, StandardCharsets.UTF_8); + + Process process = new ProcessBuilder( + pythonExecutable, bytecoderPath, input.path.toString(), output.path.toString()) + .redirectErrorStream(true) + .start(); + String consoleOutput = new String(process.getInputStream().readAllBytes(), StandardCharsets.UTF_8); + int exitCode = process.waitFor(); + if (exitCode != 0) { + throw new RuntimeException("bdd-bytecoder.py failed for '" + serviceLabel + + "' (exit " + exitCode + "):\n" + consoleOutput); + } + byte[] bytecode = Files.readAllBytes(output.path); + if (bytecode.length == 0) { + throw new RuntimeException("bdd-bytecoder.py produced an empty blob for '" + serviceLabel + "'"); + } + return bytecode; + } catch (IOException | InterruptedException e) { + throw new RuntimeException("Failed to run bdd-bytecoder.py for '" + serviceLabel + "'", e); + } + } + + /** A temp file that deletes itself on close, so {@link #compile} can lean on try-with-resources. */ + private static final class TempFile implements AutoCloseable { + private final Path path; + + TempFile(String prefix, String suffix) throws IOException { + this.path = Files.createTempFile(prefix, suffix); + } + + @Override + public void close() throws IOException { + Files.deleteIfExists(path); + } + } +} diff --git a/tools/code-generation/smithy/cpp-codegen/smithy-cpp-codegen/src/main/java/com/amazonaws/util/awsclientsmithygenerator/generators/endpointrules/EndpointRulesCodegenPlugin.java b/tools/code-generation/smithy/cpp-codegen/smithy-cpp-codegen/src/main/java/com/amazonaws/util/awsclientsmithygenerator/generators/endpointrules/EndpointRulesCodegenPlugin.java new file mode 100644 index 000000000000..110c5d0a3caa --- /dev/null +++ b/tools/code-generation/smithy/cpp-codegen/smithy-cpp-codegen/src/main/java/com/amazonaws/util/awsclientsmithygenerator/generators/endpointrules/EndpointRulesCodegenPlugin.java @@ -0,0 +1,120 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ +package com.amazonaws.util.awsclientsmithygenerator.generators.endpointrules; + +import com.amazonaws.util.awsclientsmithygenerator.generators.CppWriterDelegator; +import com.amazonaws.util.awsclientsmithygenerator.generators.ServiceNameUtil; +import software.amazon.smithy.build.PluginContext; +import software.amazon.smithy.build.SmithyBuildPlugin; +import software.amazon.smithy.model.Model; +import software.amazon.smithy.model.node.Node; +import software.amazon.smithy.model.node.ObjectNode; +import software.amazon.smithy.model.node.StringNode; +import software.amazon.smithy.model.shapes.ServiceShape; +import software.amazon.smithy.model.shapes.ShapeId; + +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; + +/** + * Generates the {@code EndpointRules.{h,cpp}} pair carrying the compiled BDD bytecode blob. + * + *

This is the smithy-side counterpart to the C2J {@code --skip-endpoint-rules-blob} flag: when that + * flag is set the legacy generator deliberately emits nothing for these two files, leaving this plugin + * to write them whole. The emitted ABI is identical to the JSON path (see {@link EndpointRulesRenderer}); + * only the blob contents change from ruleset JSON to binary BDD bytecode. + * + *

The BDD is read from the {@code smithy.rules#endpointBdd} trait on the service shape and compiled by + * shelling out to the in-tree {@code bdd-bytecoder.py}. The interpreter and compiler paths are supplied via + * the {@code pythonExecutable} and {@code bddBytecoderPath} plugin settings (threaded in from + * {@code smithy_cpp_gen.py}) because gradle's working directory cannot reach the compiler under {@code crt/}. + */ +public class EndpointRulesCodegenPlugin implements SmithyBuildPlugin { + + private static final ShapeId ENDPOINT_BDD_TRAIT = ShapeId.from("smithy.rules#endpointBdd"); + private static final String DEFAULT_PYTHON = "python3"; + + @Override + public String getName() { + return "smithy-cpp-codegen-endpoint-rules"; + } + + @Override + public void execute(PluginContext context) { + Model model = context.getModel(); + + // Mock projections stand in for legacy services with no Smithy model, hence no BDD to compile. + if (context.getProjectionName().endsWith(".mock")) { + return; + } + + ObjectNode settings = context.getSettings(); + Map serviceMap = parseMapSetting(settings, "c2jMap"); + Map namespaceMap = parseNamespaceMap(settings); + String pythonExecutable = settings.getStringMemberOrDefault("pythonExecutable", DEFAULT_PYTHON); + String bddBytecoderPath = settings.getStringMember("bddBytecoderPath") + .map(n -> n.getValue()) + .orElseThrow(() -> new IllegalStateException( + "endpoint-rules plugin requires the 'bddBytecoderPath' setting (path to bdd-bytecoder.py)")); + + CppWriterDelegator writerDelegator = new CppWriterDelegator(context.getFileManifest()); + + // Services without an endpointBdd trait keep resolving via the JSON path, so skip them here. + model.getServiceShapes().stream() + .map(service -> ServiceNameUtil.processS3CrtProjection(service, context.getProjectionName())) + .filter(service -> service.hasTrait(ENDPOINT_BDD_TRAIT)) + .forEach(service -> generateEndpointRules( + service, serviceMap, namespaceMap, pythonExecutable, bddBytecoderPath, writerDelegator)); + + writerDelegator.flushWriters(); + } + + private void generateEndpointRules(ServiceShape service, Map serviceMap, + Map namespaceMap, String pythonExecutable, + String bddBytecoderPath, CppWriterDelegator writerDelegator) { + String smithyServiceName = ServiceNameUtil.getSmithyServiceName(service, serviceMap); + String localMacro = ServiceNameUtil.getLocalMacro(service, serviceMap); + String namespace = namespaceMap.getOrDefault(smithyServiceName, ServiceNameUtil.getServiceName(service)); + String classPrefix = Optional.ofNullable(namespaceMap.get(smithyServiceName)) + .map(ServiceNameUtil::capitalize) + .orElse(ServiceNameUtil.getServiceNameUpperCamel(service)); + + String traitJson = Node.printJson(service.findTrait(ENDPOINT_BDD_TRAIT).orElseThrow().toNode()); + byte[] bytecode = BddBytecoder.compile(pythonExecutable, bddBytecoderPath, traitJson, smithyServiceName); + + writerDelegator.useFileWriter( + "include/aws/" + smithyServiceName + "/internal/" + classPrefix + "EndpointRules.h", + writer -> EndpointRulesRenderer.renderHeader(writer, namespace, classPrefix, smithyServiceName, localMacro)); + writerDelegator.useFileWriter( + "source/" + classPrefix + "EndpointRules.cpp", + writer -> EndpointRulesRenderer.renderSource(writer, namespace, classPrefix, smithyServiceName, bytecode)); + } + + private Map parseMapSetting(ObjectNode settings, String key) { + return settings.getMember(key) + .filter(Node::isStringNode) + .map(Node::expectStringNode) + .map(StringNode::getValue) + .map(Node::parseJsonWithComments) + .map(Node::expectObjectNode) + .map(mapNode -> mapNode.getMembers().entrySet().stream() + .collect(Collectors.toMap( + entry -> entry.getKey().getValue(), + entry -> entry.getValue().expectStringNode().getValue()))) + .orElse(Map.of()); + } + + private Map parseNamespaceMap(ObjectNode settings) { + return settings.getMember("namespaceMappings") + .map(node -> node.expectStringNode().getValue()) + .map(jsonStr -> Node.parseJsonWithComments(jsonStr).expectObjectNode()) + .map(node -> node.getMembers().entrySet().stream() + .collect(Collectors.toMap( + entry -> entry.getKey().getValue(), + entry -> ServiceNameUtil.sanitizeServiceAbbreviation(entry.getValue().expectStringNode().getValue())))) + .orElse(Map.of()); + } +} diff --git a/tools/code-generation/smithy/cpp-codegen/smithy-cpp-codegen/src/main/java/com/amazonaws/util/awsclientsmithygenerator/generators/endpointrules/EndpointRulesRenderer.java b/tools/code-generation/smithy/cpp-codegen/smithy-cpp-codegen/src/main/java/com/amazonaws/util/awsclientsmithygenerator/generators/endpointrules/EndpointRulesRenderer.java new file mode 100644 index 000000000000..4f68ff2b822f --- /dev/null +++ b/tools/code-generation/smithy/cpp-codegen/smithy-cpp-codegen/src/main/java/com/amazonaws/util/awsclientsmithygenerator/generators/endpointrules/EndpointRulesRenderer.java @@ -0,0 +1,86 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ +package com.amazonaws.util.awsclientsmithygenerator.generators.endpointrules; + +import com.amazonaws.util.awsclientsmithygenerator.generators.CppWriter; + +public final class EndpointRulesRenderer { + + private static final int BYTES_PER_LINE = 25; + + private EndpointRulesRenderer() {} + + public static void renderHeader(CppWriter writer, String namespace, String classPrefix, + String smithyServiceName, String localMacro) { + writer.write("/**"); + writer.write(" * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved."); + writer.write(" * SPDX-License-Identifier: Apache-2.0."); + writer.write(" */"); + writer.write(""); + writer.write("#pragma once"); + writer.write("#include ", smithyServiceName, classPrefix); + writer.write(""); + writer.write("#include "); + writer.write(""); + writer.write("namespace Aws {"); + writer.write("namespace $L {", namespace); + writer.write("class $1L $2LEndpointRules {", localMacro, classPrefix); + writer.write(" public:"); + writer.write(" static const size_t RulesBlobStrLen;"); + writer.write(" static const size_t RulesBlobSize;"); + writer.write(""); + writer.write(" static const char* GetRulesBlob();"); + writer.write("};"); + writer.write("} // namespace $L", namespace); + writer.write("} // namespace Aws"); + } + + public static void renderSource(CppWriter writer, String namespace, String classPrefix, + String smithyServiceName, byte[] bytecode) { + writer.write("/**"); + writer.write(" * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved."); + writer.write(" * SPDX-License-Identifier: Apache-2.0."); + writer.write(" */"); + writer.write(""); + writer.write("#include "); + writer.write("#include ", smithyServiceName, classPrefix); + writer.write(""); + writer.write("namespace Aws {"); + writer.write("namespace $L {", namespace); + writer.write("const size_t $1LEndpointRules::RulesBlobStrLen = $2L;", classPrefix, Integer.toString(bytecode.length)); + writer.write("const size_t $1LEndpointRules::RulesBlobSize = $2L;", classPrefix, Integer.toString(bytecode.length)); + writer.write(""); + writer.write("using RulesBlobT = Aws::Array;", classPrefix); + writer.write("static constexpr RulesBlobT RulesBlob = {"); + writeByteArray(writer, bytecode); + writer.write("};"); + writer.write(""); + writer.write("const char* $LEndpointRules::GetRulesBlob() { return RulesBlob.data(); }", classPrefix); + writer.write("} // namespace $L", namespace); + writer.write("} // namespace Aws"); + } + + private static void writeByteArray(CppWriter writer, byte[] bytecode) { + StringBuilder line = new StringBuilder(" {"); + for (int i = 0; i < bytecode.length; i++) { + line.append(charLiteral(bytecode[i])); + if (i != bytecode.length - 1) { + line.append(','); + } + if ((i + 1) % BYTES_PER_LINE == 0 && i != bytecode.length - 1) { + writer.write("$L", line.toString()); + line.setLength(0); + line.append(" "); + } + } + line.append("}"); + writer.write("$L", line.toString()); + } + + private static String charLiteral(byte b) { + return String.format("'\\x%02x'", b & 0xFF); + } + +} diff --git a/tools/code-generation/smithy/cpp-codegen/smithy-cpp-codegen/src/main/resources/META-INF/services/software.amazon.smithy.build.SmithyBuildPlugin b/tools/code-generation/smithy/cpp-codegen/smithy-cpp-codegen/src/main/resources/META-INF/services/software.amazon.smithy.build.SmithyBuildPlugin index 1f36b05b7f27..85ed5c1ae395 100644 --- a/tools/code-generation/smithy/cpp-codegen/smithy-cpp-codegen/src/main/resources/META-INF/services/software.amazon.smithy.build.SmithyBuildPlugin +++ b/tools/code-generation/smithy/cpp-codegen/smithy-cpp-codegen/src/main/resources/META-INF/services/software.amazon.smithy.build.SmithyBuildPlugin @@ -1,3 +1,4 @@ com.amazonaws.util.awsclientsmithygenerator.generators.pagination.PaginationCodegenPlugin com.amazonaws.util.awsclientsmithygenerator.generators.waiters.WaiterCodegenPlugin com.amazonaws.util.awsclientsmithygenerator.generators.model.ModelCodegenPlugin +com.amazonaws.util.awsclientsmithygenerator.generators.endpointrules.EndpointRulesCodegenPlugin diff --git a/tools/scripts/codegen/legacy_c2j_cpp_gen.py b/tools/scripts/codegen/legacy_c2j_cpp_gen.py index 634d1d7ce50f..0a9d765329d1 100644 --- a/tools/scripts/codegen/legacy_c2j_cpp_gen.py +++ b/tools/scripts/codegen/legacy_c2j_cpp_gen.py @@ -19,6 +19,7 @@ from codegen.include_tests_util import IncludeTestsUtil from codegen.model_utils import ServiceModel +from codegen.smithy_cpp_gen import bdd_endpoint_services SMITHY_SUPPORTED_CLIENTS = [ "dynamodb", @@ -60,6 +61,8 @@ def __init__(self, args: dict, c2j_models: dict, skip_model_services: set = None self.debug = args.get("debug", False) self.c2j_models = c2j_models self.skip_model_services = skip_model_services or set() + self.use_smithy_bdd_endpoints = args.get("use_smithy_bdd_endpoints", False) + self.bdd_endpoint_services = bdd_endpoint_services() if self.use_smithy_bdd_endpoints else set() generator_location = args["path_to_generator"] or DEFAULT_GENERATOR_LOCATION generator_location = str(Path(generator_location).absolute()) @@ -198,6 +201,11 @@ def _init_common_java_cli(self, if self.debug: print(f" Skipping C2J model generation for {service_name} (using Smithy models)") + if service_name in self.bdd_endpoint_services: + run_command += ["--skip-endpoint-rules-blob"] + if self.debug: + print(f" Skipping C2J endpoint-rules blob for {service_name} (using Smithy BDD endpoints)") + for key, val in kwargs.items(): run_command += [f"--{key}", val] diff --git a/tools/scripts/codegen/smithy_cpp_gen.py b/tools/scripts/codegen/smithy_cpp_gen.py index 7dfc8b77c444..29e0fdc3b318 100644 --- a/tools/scripts/codegen/smithy_cpp_gen.py +++ b/tools/scripts/codegen/smithy_cpp_gen.py @@ -10,21 +10,56 @@ import os import shutil import subprocess +import sys +from pathlib import Path from typing import List SMITHY_GENERATOR_LOCATION = "tools/code-generation/smithy/cpp-codegen" SMITHY_TO_C2J_MAP_FILE = "tools/code-generation/smithy/cpp-codegen/smithy2c2j_service_map.json" SMITHY_NAMPESPACE_MAPPING_FILE = "tools/code-generation/smithy/mapping/smithy-namespace-mapping.json" +SMITHY_API_DESCRIPTIONS_DIR = "tools/code-generation/smithy/api-descriptions" +BDD_BYTECODER_FILE = "crt/aws-crt-cpp/crt/aws-c-sdkutils/compiler/bdd-bytecoder.py" +ENDPOINT_BDD_TRAIT = "smithy.rules#endpointBdd" + + +def bdd_endpoint_services(): + """C2J service names whose Smithy model carries the endpointBdd trait on its service shape, + i.e. the services the Smithy generator will emit a BDD blob for. C2J must skip its JSON + endpoint-rules blob for exactly this set; services outside it (legacy/mock, or models lacking + the trait) keep the JSON path. This MUST stay in sync with EndpointRulesCodegenPlugin's + service.hasTrait(ENDPOINT_BDD_TRAIT) check — divergence causes a missing GetRulesBlob() symbol.""" + with open(os.path.abspath(SMITHY_TO_C2J_MAP_FILE), 'r') as file: + smithy_to_c2j = json.load(file) + descriptions_dir = Path(os.path.abspath(SMITHY_API_DESCRIPTIONS_DIR)) + services = { + smithy_to_c2j.get(model.stem, model.stem) + for model in descriptions_dir.glob("*.json") + if _has_bdd_service_shape(model) + } + if "s3" in services: + services.add("s3-crt") + return services + + +def _has_bdd_service_shape(model_path): + """True if any service shape in the Smithy model carries the endpointBdd trait.""" + shapes = json.loads(model_path.read_text()).get("shapes", {}).values() + return any( + shape.get("type") == "service" and ENDPOINT_BDD_TRAIT in shape.get("traits", {}) + for shape in shapes + ) class SmithyCppGen(object): """Wrapper for Smithy C++ code generator for C++ SDK""" def __init__(self, debug: bool, use_smithy_models: bool = False, - smithy_model_services: set = None, **kwargs): + smithy_model_services: set = None, use_smithy_bdd_endpoints: bool = False, + **kwargs): self.debug = debug self.use_smithy_models = use_smithy_models self.smithy_model_services = smithy_model_services or set() + self.use_smithy_bdd_endpoints = use_smithy_bdd_endpoints with open(os.path.abspath(SMITHY_TO_C2J_MAP_FILE), 'r') as file: self.smithy_c2j_data = json.load(file) self.c2j_smithy_data = {value: key for key, value in self.smithy_c2j_data.items()} @@ -44,6 +79,10 @@ def generate(self, clients_to_build: set): plugins_to_copy.append("smithy-cpp-codegen-models") if self.debug: print(f"Including Smithy model files for: {sorted(self.smithy_model_services)}") + if self.use_smithy_bdd_endpoints: + plugins_to_copy.append("smithy-cpp-codegen-endpoint-rules") + if self.debug: + print("Including Smithy-generated BDD endpoint-rules blob files") self._copy_cpp_codegen_contents( os.path.abspath("tools/code-generation/smithy/cpp-codegen"), plugins_to_copy, @@ -62,6 +101,10 @@ def _generate_pagination(self, smithy_services: List[str], smithy_c2j_data: str) ] if self.use_smithy_models: smithy_codegen_command.append("-PgenerateModels=true") + if self.use_smithy_bdd_endpoints: + smithy_codegen_command.append("-PgenerateEndpointRules=true") + smithy_codegen_command.append("-PbddBytecoderPath=" + os.path.abspath(BDD_BYTECODER_FILE)) + smithy_codegen_command.append("-PpythonExecutable=" + sys.executable) try: if self.debug: diff --git a/tools/scripts/run_code_generation.py b/tools/scripts/run_code_generation.py index 01f86ecfbb99..7d65f7abaf63 100644 --- a/tools/scripts/run_code_generation.py +++ b/tools/scripts/run_code_generation.py @@ -81,6 +81,12 @@ def parse_arguments() -> dict: help="Comma-separated list of services to generate models from Smithy. " "Only effective with --use-smithy-models. " "Defaults to all services in --client_list if omitted.") + parser.add_argument("--use-smithy-bdd-endpoints", + help="Use Smithy-generated BDD bytecode for endpoint resolution instead of the C2J " + "JSON ruleset. C2J skips the JSON endpoint-rules blob and emits a BDD-backed " + "provider; Smithy codegen produces the compiled bytecode blob. When omitted, " + "endpoint resolution keeps using the C2J JSON path.", + action="store_true") args = vars(parser.parse_args()) arg_map = {"debug": args.get("debug", False)} @@ -142,6 +148,7 @@ def parse_arguments() -> dict: arg_map["generate_protocol_tests"] = args.get("generate_protocol_tests", None) arg_map["generate_install_tests"] = args.get("generate_install_tests", None) arg_map["use_smithy_models"] = args.get("use_smithy_models", False) + arg_map["use_smithy_bdd_endpoints"] = args.get("use_smithy_bdd_endpoints", False) smithy_model_services_raw = args.get("smithy_model_services", None) if smithy_model_services_raw: arg_map["smithy_model_services"] = set(smithy_model_services_raw.replace(";", ",").split(",")) @@ -205,7 +212,8 @@ def main(): if clients_to_build: smithy_cpp_gen = SmithyCppGen(args["debug"], use_smithy_models=bool(smithy_model_services), - smithy_model_services=smithy_model_services) + smithy_model_services=smithy_model_services, + use_smithy_bdd_endpoints=args.get("use_smithy_bdd_endpoints", False)) if smithy_cpp_gen.generate(clients_to_build) != 0: print("ERROR: Failed to generate Smithy code!") return -1