Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commit47158f2

Browse files
authored
s2: Add support for custom stream encoder (#755)
```Go// WriterCustomEncoder allows to override the encoder for blocks on the stream.// The function must compress 'src' into 'dst' and return the bytes used in dst as an integer.// Block size (initial varint) should not be added by the encoder.// Returning value 0 indicates the block could not be compressed.// The function should expect to be called concurrently.func WriterCustomEncoder(fn func(dst, src []byte) int) WriterOption```
1 parentfdc8ab0 commit47158f2

File tree

3 files changed

+43
-4
lines changed

3 files changed

+43
-4
lines changed

‎internal/snapref/encode_other.go‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -103,6 +103,28 @@ func hash(u, shift uint32) uint32 {
103103
return (u*0x1e35a7bd)>>shift
104104
}
105105

106+
// EncodeBlockInto exposes encodeBlock but checks dst size.
107+
funcEncodeBlockInto(dst,src []byte) (dint) {
108+
ifMaxEncodedLen(len(src))>len(dst) {
109+
return0
110+
}
111+
112+
// encodeBlock breaks on too big blocks, so split.
113+
forlen(src)>0 {
114+
p:=src
115+
src=nil
116+
iflen(p)>maxBlockSize {
117+
p,src=p[:maxBlockSize],p[maxBlockSize:]
118+
}
119+
iflen(p)<minNonLiteralBlockSize {
120+
d+=emitLiteral(dst[d:],p)
121+
}else {
122+
d+=encodeBlock(dst[d:],p)
123+
}
124+
}
125+
returnd
126+
}
127+
106128
// encodeBlock encodes a non-empty src to a guaranteed-large-enough dst. It
107129
// assumes that the varint-encoded length of the decompressed bytes has already
108130
// been written.

‎s2/encode.go‎

Lines changed: 20 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -430,10 +430,11 @@ type Writer struct {
430430
buffers sync.Pool
431431
padint
432432

433-
writer io.Writer
434-
randSrc io.Reader
435-
writerWg sync.WaitGroup
436-
indexIndex
433+
writer io.Writer
434+
randSrc io.Reader
435+
writerWg sync.WaitGroup
436+
indexIndex
437+
customEncfunc(dst,src []byte)int
437438

438439
// wroteStreamHeader is whether we have written the stream header.
439440
wroteStreamHeaderbool
@@ -799,6 +800,9 @@ func (w *Writer) EncodeBuffer(buf []byte) (err error) {
799800
}
800801

801802
func (w*Writer)encodeBlock(obuf,uncompressed []byte)int {
803+
ifw.customEnc!=nil {
804+
returnw.customEnc(obuf,uncompressed)
805+
}
802806
ifw.snappy {
803807
switchw.level {
804808
caselevelFast:
@@ -1365,3 +1369,15 @@ func WriterFlushOnWrite() WriterOption {
13651369
returnnil
13661370
}
13671371
}
1372+
1373+
// WriterCustomEncoder allows to override the encoder for blocks on the stream.
1374+
// The function must compress 'src' into 'dst' and return the bytes used in dst as an integer.
1375+
// Block size (initial varint) should not be added by the encoder.
1376+
// Returning value 0 indicates the block could not be compressed.
1377+
// The function should expect to be called concurrently.
1378+
funcWriterCustomEncoder(fnfunc(dst,src []byte)int)WriterOption {
1379+
returnfunc(w*Writer)error {
1380+
w.customEnc=fn
1381+
returnnil
1382+
}
1383+
}

‎s2/encode_test.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,7 @@ func testOptions(t testing.TB) map[string][]WriterOption {
5959
forname,opt:=rangetestOptions {
6060
x[name]=opt
6161
x[name+"-snappy"]=cloneAdd(opt,WriterSnappyCompat())
62+
x[name+"-custom"]=cloneAdd(opt,WriterCustomEncoder(snapref.EncodeBlockInto))
6263
}
6364
testOptions=x
6465
returntestOptions

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp