Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commit690fcaa

Browse files
wty-BryantTianyi Wang
and
Tianyi Wang
authored
Add Gzip request compression feature (#467)
* Add and Merge request compression feature* Modify and Merge request compression codegen* Add and Merge changelog for last commit* Modify logic of request compression middleware* Add request compression algorithm codegen part* resolve METAINFO conflict* Change dependency format* Revert dependency format* Change request compression middleware to operation level* Change codegen comment* Change static middleware import* Change go dependency codegen* Add body compare fn to request compress op unit test* Solve rebase conflict---------Co-authored-by: Tianyi Wang <wty@amazon.com>
1 parent88d16be commit690fcaa

File tree

12 files changed

+602
-1
lines changed

12 files changed

+602
-1
lines changed
Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
{
2+
"id":"80ed2832-7bcd-4301-a264-f318efaf8216",
3+
"type":"feature",
4+
"description":"Support modeled request compression.",
5+
"modules": [
6+
"."
7+
]
8+
}

‎codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/SmithyGoDependency.java

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,8 @@ public final class SmithyGoDependency {
5353
publicstaticfinalGoDependencySMITHY_HTTP_TRANSPORT =smithy("transport/http","smithyhttp");
5454
publicstaticfinalGoDependencySMITHY_MIDDLEWARE =smithy("middleware");
5555
publicstaticfinalGoDependencySMITHY_PRIVATE_PROTOCOL =smithy("private/protocol","smithyprivateprotocol");
56+
publicstaticfinalGoDependencySMITHY_REQUEST_COMPRESSION =
57+
smithy("private/requestcompression","smithyrequestcompression");
5658
publicstaticfinalGoDependencySMITHY_TIME =smithy("time","smithytime");
5759
publicstaticfinalGoDependencySMITHY_HTTP_BINDING =smithy("encoding/httpbinding");
5860
publicstaticfinalGoDependencySMITHY_JSON =smithy("encoding/json","smithyjson");

‎codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/SmithyGoTypes.java

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -105,4 +105,11 @@ public static final class Bearer {
105105
publicstaticfinalSymbolNewSignHTTPSMessage =SmithyGoDependency.SMITHY_AUTH_BEARER.valueSymbol("NewSignHTTPSMessage");
106106
}
107107
}
108+
109+
publicstaticfinalclassPrivate {
110+
publicstaticfinalclassRequestCompression {
111+
publicstaticfinalSymbolAddRequestCompression =SmithyGoDependency.SMITHY_REQUEST_COMPRESSION.valueSymbol("AddRequestCompression");
112+
publicstaticfinalSymbolAddCaptureUncompressedRequest =SmithyGoDependency.SMITHY_REQUEST_COMPRESSION.valueSymbol("AddCaptureUncompressedRequestMiddleware");
113+
}
114+
}
108115
}

‎codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/integration/HttpProtocolUnitTestRequestGenerator.java

Lines changed: 51 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,20 +17,31 @@
1717

1818
packagesoftware.amazon.smithy.go.codegen.integration;
1919

20+
importstaticsoftware.amazon.smithy.go.codegen.GoWriter.goTemplate;
21+
importstaticsoftware.amazon.smithy.go.codegen.SmithyGoTypes.Private.RequestCompression.AddCaptureUncompressedRequest;
22+
23+
importjava.util.Arrays;
24+
importjava.util.HashSet;
25+
importjava.util.Set;
2026
importjava.util.function.Consumer;
2127
importjava.util.logging.Logger;
2228
importsoftware.amazon.smithy.codegen.core.Symbol;
2329
importsoftware.amazon.smithy.go.codegen.GoWriter;
2430
importsoftware.amazon.smithy.go.codegen.SmithyGoDependency;
31+
importsoftware.amazon.smithy.go.codegen.SmithyGoTypes;
2532
importsoftware.amazon.smithy.go.codegen.SymbolUtils;
33+
importsoftware.amazon.smithy.model.traits.RequestCompressionTrait;
2634
importsoftware.amazon.smithy.protocoltests.traits.HttpRequestTestCase;
35+
importsoftware.amazon.smithy.utils.MapUtils;
2736

2837
/**
2938
* Generates HTTP protocol unit tests for HTTP request test cases.
3039
*/
3140
publicclassHttpProtocolUnitTestRequestGeneratorextendsHttpProtocolUnitTestGenerator<HttpRequestTestCase> {
3241
privatestaticfinalLoggerLOGGER =Logger.getLogger(HttpProtocolUnitTestRequestGenerator.class.getName());
3342

43+
privatestaticfinalSet<String>ALLOWED_ALGORITHMS =newHashSet<>(Arrays.asList("gzip"));
44+
3445
/**
3546
* Initializes the protocol test generator.
3647
*
@@ -198,6 +209,10 @@ protected void generateTestCaseValues(GoWriter writer, HttpRequestTestCase testC
198209
*/
199210
protectedvoidgenerateTestBodySetup(GoWriterwriter) {
200211
writer.write("actualReq := &http.Request{}");
212+
if (operation.hasTrait(RequestCompressionTrait.class)) {
213+
writer.addUseImports(SmithyGoDependency.BYTES);
214+
writer.write("rawBodyBuf := &bytes.Buffer{}");
215+
}
201216
}
202217

203218
/**
@@ -227,8 +242,29 @@ protected void generateTestInvokeClientOperation(GoWriter writer, String clientN
227242
writer.write("return $T(stack, actualReq)",
228243
SymbolUtils.createValueSymbolBuilder("AddCaptureRequestMiddleware",
229244
SmithyGoDependency.SMITHY_PRIVATE_PROTOCOL).build());
230-
});
245+
});
246+
if (operation.hasTrait(RequestCompressionTrait.class)) {
247+
writer.write(goTemplate("""
248+
options.APIOptions = append(options.APIOptions, func(stack $stack:P) error {
249+
return $captureRequest:T(stack, rawBodyBuf)
250+
})
251+
""",
252+
MapUtils.of(
253+
"stack",SmithyGoTypes.Middleware.Stack,
254+
"captureRequest",AddCaptureUncompressedRequest
255+
)));
256+
}
231257
});
258+
259+
if (operation.hasTrait(RequestCompressionTrait.class)) {
260+
writer.write(goTemplate("""
261+
disable := $client:L.Options().DisableRequestCompression
262+
min := $client:L.Options().RequestMinCompressSizeBytes
263+
""",
264+
MapUtils.of(
265+
"client",clientName
266+
)));
267+
}
232268
}
233269

234270
/**
@@ -259,6 +295,20 @@ protected void generateTestAssertions(GoWriter writer) {
259295
writer.write("t.Errorf(\"expect body equal, got %v\", err)");
260296
});
261297
});
298+
299+
if (operation.hasTrait(RequestCompressionTrait.class)) {
300+
Stringalgorithm =operation.expectTrait(RequestCompressionTrait.class).getEncodings()
301+
.stream().filter(it ->ALLOWED_ALGORITHMS.contains(it)).findFirst().get();
302+
writer.write(goTemplate("""
303+
if err := smithytesting.CompareCompressedBytes(rawBodyBuf, actualReq.Body,
304+
disable, min, $algorithm:S); err != nil {
305+
t.Errorf("unzipped request body not match: %q", err)
306+
}
307+
""",
308+
MapUtils.of(
309+
"algorithm",algorithm
310+
)));
311+
}
262312
}
263313

264314
publicstaticclassBuilderextendsHttpProtocolUnitTestGenerator.Builder<HttpRequestTestCase> {
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,160 @@
1+
/*
2+
* Copyright 2023 Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License").
5+
* You may not use this file except in compliance with the License.
6+
* A copy of the License is located at
7+
*
8+
* http://aws.amazon.com/apache2.0
9+
*
10+
* or in the "license" file accompanying this file. This file is distributed
11+
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
12+
* express or implied. See the License for the specific language governing
13+
* permissions and limitations under the License.
14+
*/
15+
16+
packagesoftware.amazon.smithy.go.codegen.requestcompression;
17+
18+
importstaticsoftware.amazon.smithy.go.codegen.GoWriter.goTemplate;
19+
20+
importjava.util.ArrayList;
21+
importjava.util.List;
22+
importsoftware.amazon.smithy.codegen.core.SymbolProvider;
23+
importsoftware.amazon.smithy.go.codegen.GoCodegenPlugin;
24+
importsoftware.amazon.smithy.go.codegen.GoDelegator;
25+
importsoftware.amazon.smithy.go.codegen.GoSettings;
26+
importsoftware.amazon.smithy.go.codegen.GoUniverseTypes;
27+
importsoftware.amazon.smithy.go.codegen.GoWriter;
28+
importsoftware.amazon.smithy.go.codegen.SmithyGoTypes;
29+
importsoftware.amazon.smithy.go.codegen.SymbolUtils;
30+
importsoftware.amazon.smithy.go.codegen.integration.ConfigField;
31+
importsoftware.amazon.smithy.go.codegen.integration.GoIntegration;
32+
importsoftware.amazon.smithy.go.codegen.integration.MiddlewareRegistrar;
33+
importsoftware.amazon.smithy.go.codegen.integration.RuntimeClientPlugin;
34+
importsoftware.amazon.smithy.model.Model;
35+
importsoftware.amazon.smithy.model.knowledge.TopDownIndex;
36+
importsoftware.amazon.smithy.model.shapes.OperationShape;
37+
importsoftware.amazon.smithy.model.shapes.ServiceShape;
38+
importsoftware.amazon.smithy.model.shapes.ShapeId;
39+
importsoftware.amazon.smithy.model.traits.RequestCompressionTrait;
40+
importsoftware.amazon.smithy.utils.ListUtils;
41+
importsoftware.amazon.smithy.utils.MapUtils;
42+
43+
44+
publicfinalclassRequestCompressionimplementsGoIntegration {
45+
privatestaticfinalStringDISABLE_REQUEST_COMPRESSION ="DisableRequestCompression";
46+
47+
privatestaticfinalStringREQUEST_MIN_COMPRESSION_SIZE_BYTES ="RequestMinCompressSizeBytes";
48+
49+
privatefinalList<RuntimeClientPlugin>runtimeClientPlugins =newArrayList<>();
50+
51+
// Write operation plugin for request compression middleware
52+
@Override
53+
publicvoidprocessFinalizedModel(GoSettingssettings,Modelmodel) {
54+
ServiceShapeservice =settings.getService(model);
55+
TopDownIndex.of(model)
56+
.getContainedOperations(service).forEach(operation -> {
57+
if (!operation.hasTrait(RequestCompressionTrait.class)) {
58+
return;
59+
}
60+
SymbolProvidersymbolProvider =GoCodegenPlugin.createSymbolProvider(model,settings);
61+
StringfuncName =getAddRequestCompressionMiddlewareFuncName(
62+
symbolProvider.toSymbol(operation).getName()
63+
);
64+
runtimeClientPlugins.add(RuntimeClientPlugin.builder().operationPredicate((m,s,o) -> {
65+
if (!o.hasTrait(RequestCompressionTrait.class)) {
66+
returnfalse;
67+
}
68+
returno.equals(operation);
69+
}).registerMiddleware(MiddlewareRegistrar.builder()
70+
.resolvedFunction(SymbolUtils.buildPackageSymbol(funcName))
71+
.useClientOptions().build())
72+
.build());
73+
});
74+
}
75+
76+
@Override
77+
publicvoidwriteAdditionalFiles(
78+
GoSettingssettings,
79+
Modelmodel,
80+
SymbolProvidersymbolProvider,
81+
GoDelegatorgoDelegator
82+
) {
83+
ServiceShapeservice =settings.getService(model);
84+
for (ShapeIdoperationID :service.getAllOperations()) {
85+
OperationShapeoperation =model.expectShape(operationID,OperationShape.class);
86+
if (!operation.hasTrait(RequestCompressionTrait.class)) {
87+
continue;
88+
}
89+
goDelegator.useShapeWriter(operation,writeMiddlewareHelper(symbolProvider,operation));
90+
}
91+
}
92+
93+
94+
publicstaticbooleanisRequestCompressionService(Modelmodel,ServiceShapeservice) {
95+
returnTopDownIndex.of(model)
96+
.getContainedOperations(service).stream()
97+
.anyMatch(it ->it.hasTrait(RequestCompressionTrait.class));
98+
}
99+
100+
@Override
101+
publicList<RuntimeClientPlugin>getClientPlugins() {
102+
runtimeClientPlugins.add(
103+
RuntimeClientPlugin.builder()
104+
.servicePredicate(RequestCompression::isRequestCompressionService)
105+
.configFields(ListUtils.of(
106+
ConfigField.builder()
107+
.name(DISABLE_REQUEST_COMPRESSION)
108+
.type(GoUniverseTypes.Bool)
109+
.documentation(
110+
"Whether to disable automatic request compression for supported operations.")
111+
.build(),
112+
ConfigField.builder()
113+
.name(REQUEST_MIN_COMPRESSION_SIZE_BYTES)
114+
.type(GoUniverseTypes.Int64)
115+
.documentation("The minimum request body size, in bytes, at which compression "
116+
+"should occur. The default value is 10 KiB. Values must fall within "
117+
+"[0, 1MiB].")
118+
.build()
119+
))
120+
.build()
121+
);
122+
123+
returnruntimeClientPlugins;
124+
}
125+
126+
privateGoWriter.WritablegenerateAlgorithmList(List<String>algorithms) {
127+
returngoTemplate("""
128+
[]string{
129+
$W
130+
}
131+
""",
132+
GoWriter.ChainWritable.of(
133+
algorithms.stream()
134+
.map(it ->goTemplate("$S,",it))
135+
.toList()
136+
).compose(false));
137+
}
138+
139+
privatestaticStringgetAddRequestCompressionMiddlewareFuncName(StringoperationName) {
140+
returnString.format("addOperation%sRequestCompressionMiddleware",operationName);
141+
}
142+
143+
privateGoWriter.WritablewriteMiddlewareHelper(SymbolProvidersymbolProvider,OperationShapeoperation) {
144+
StringoperationName =symbolProvider.toSymbol(operation).getName();
145+
RequestCompressionTraittrait =operation.expectTrait(RequestCompressionTrait.class);
146+
147+
returngoTemplate("""
148+
func $add:L(stack $stack:P, options Options) error {
149+
return $addInternal:T(stack, options.DisableRequestCompression, options.RequestMinCompressSizeBytes,
150+
$algorithms:W)
151+
}
152+
""",
153+
MapUtils.of(
154+
"add",getAddRequestCompressionMiddlewareFuncName(operationName),
155+
"stack",SmithyGoTypes.Middleware.Stack,
156+
"addInternal",SmithyGoTypes.Private.RequestCompression.AddRequestCompression,
157+
"algorithms",generateAlgorithmList(trait.getEncodings())
158+
));
159+
}
160+
}

‎codegen/smithy-go-codegen/src/main/resources/META-INF/services/software.amazon.smithy.go.codegen.integration.GoIntegration

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,3 +12,5 @@ software.amazon.smithy.go.codegen.endpoints.EndpointClientPluginsGenerator
1212
# modeled auth schemes
1313
software.amazon.smithy.go.codegen.integration.auth.SigV4AuthScheme
1414
software.amazon.smithy.go.codegen.integration.auth.AnonymousAuthScheme
15+
16+
software.amazon.smithy.go.codegen.requestcompression.RequestCompression

‎private/requestcompression/gzip.go

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
package requestcompression
2+
3+
import (
4+
"bytes"
5+
"compress/gzip"
6+
"fmt"
7+
"io"
8+
)
9+
10+
funcgzipCompress(input io.Reader) ([]byte,error) {
11+
varb bytes.Buffer
12+
w,err:=gzip.NewWriterLevel(&b,gzip.DefaultCompression)
13+
iferr!=nil {
14+
returnnil,fmt.Errorf("failed to create gzip writer, %v",err)
15+
}
16+
17+
inBytes,err:=io.ReadAll(input)
18+
iferr!=nil {
19+
returnnil,fmt.Errorf("failed read payload to compress, %v",err)
20+
}
21+
22+
if_,err=w.Write(inBytes);err!=nil {
23+
returnnil,fmt.Errorf("failed to write payload to be compressed, %v",err)
24+
}
25+
iferr=w.Close();err!=nil {
26+
returnnil,fmt.Errorf("failed to flush payload being compressed, %v",err)
27+
}
28+
29+
returnb.Bytes(),nil
30+
}
Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
package requestcompression
2+
3+
import (
4+
"bytes"
5+
"context"
6+
"fmt"
7+
"github.com/aws/smithy-go/middleware"
8+
smithyhttp"github.com/aws/smithy-go/transport/http"
9+
"io"
10+
"net/http"
11+
)
12+
13+
constcaptureUncompressedRequestID="CaptureUncompressedRequest"
14+
15+
// AddCaptureUncompressedRequestMiddleware captures http request before compress encoding for check
16+
funcAddCaptureUncompressedRequestMiddleware(stack*middleware.Stack,buf*bytes.Buffer)error {
17+
returnstack.Serialize.Insert(&captureUncompressedRequestMiddleware{
18+
buf:buf,
19+
},"RequestCompression",middleware.Before)
20+
}
21+
22+
typecaptureUncompressedRequestMiddlewarestruct {
23+
req*http.Request
24+
buf*bytes.Buffer
25+
bytes []byte
26+
}
27+
28+
// ID returns id of the captureUncompressedRequestMiddleware
29+
func (*captureUncompressedRequestMiddleware)ID()string {
30+
returncaptureUncompressedRequestID
31+
}
32+
33+
// HandleSerialize captures request payload before it is compressed by request compression middleware
34+
func (m*captureUncompressedRequestMiddleware)HandleSerialize(ctx context.Context,input middleware.SerializeInput,next middleware.SerializeHandler,
35+
) (
36+
output middleware.SerializeOutput,metadata middleware.Metadata,errerror,
37+
) {
38+
request,ok:=input.Request.(*smithyhttp.Request)
39+
if!ok {
40+
returnoutput,metadata,fmt.Errorf("error when retrieving http request")
41+
}
42+
43+
_,err=io.Copy(m.buf,request.GetStream())
44+
iferr!=nil {
45+
returnoutput,metadata,fmt.Errorf("error when copying http request stream: %q",err)
46+
}
47+
iferr=request.RewindStream();err!=nil {
48+
returnoutput,metadata,fmt.Errorf("error when rewinding request stream: %q",err)
49+
}
50+
51+
returnnext.HandleSerialize(ctx,input)
52+
}

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp