Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commit268fea2

Browse files
committed
Add SkipEmptyValues option for Networks iteration
Add SkipEmptyValues() NetworksOption that skips networks whose data isan empty map or empty array. This is useful for databases that storeempty maps or arrays for records without meaningful data, allowingiteration over only records with actual content.The implementation adds IsEmptyValueAt() method to ReflectionDecoderfor efficient empty value detection without modifying decoder state.Includes comprehensive tests and examples using GeoIP2-Anonymous-IPtest database which contains 522 empty maps out of 529 networks.Also improves linter configuration by disabling lll since golineshandles line length management, and configures errcheck to excludeReader.Close in tests.Closes#172.
1 parentdd159a3 commit268fea2

File tree

6 files changed

+303
-7
lines changed

6 files changed

+303
-7
lines changed

‎.golangci.yml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ linters:
1919
-gosmopolitan
2020
-inamedparam
2121
-interfacebloat
22+
-lll
2223
-mnd
2324
-nlreturn
2425
-noinlineerr
@@ -31,6 +32,9 @@ linters:
3132
-wsl
3233
-wsl_v5
3334
settings:
35+
errcheck:
36+
exclude-functions:
37+
-(*github.com/oschwald/maxminddb-golang/v2.Reader).Close
3438
errorlint:
3539
errorf:true
3640
asserts:true

‎CHANGELOG.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,10 @@
1010
databases.
1111
- Added validation for invalid prefixes in`NetworksWithin` to prevent
1212
unexpected behavior with malformed input.
13+
- Added`SkipEmptyValues()` option for`Networks` and`NetworksWithin` to skip
14+
networks whose data is an empty map or empty array. This is useful for
15+
databases that store empty maps or arrays for records without meaningful
16+
data. GitHub#172.
1317

1418
##2.0.0-beta.8 - 2025-07-15
1519

‎example_test.go‎

Lines changed: 80 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ func ExampleReader_Lookup_struct() {
1515
iferr!=nil {
1616
log.Fatal(err)
1717
}
18-
deferdb.Close()//nolint:errcheck // error doesn't matter
18+
deferdb.Close()
1919

2020
addr:=netip.MustParseAddr("81.2.69.142")
2121

@@ -40,7 +40,7 @@ func ExampleReader_Lookup_interface() {
4040
iferr!=nil {
4141
log.Fatal(err)
4242
}
43-
deferdb.Close()//nolint:errcheck // error doesn't matter
43+
deferdb.Close()
4444

4545
addr:=netip.MustParseAddr("81.2.69.142")
4646

@@ -50,7 +50,6 @@ func ExampleReader_Lookup_interface() {
5050
log.Panic(err)
5151
}
5252
fmt.Printf("%v",record)
53-
//nolint:lll
5453
// Output:
5554
// map[city:map[geoname_id:2643743 names:map[de:London en:London es:Londres fr:Londres ja:ロンドン pt-BR:Londres ru:Лондон]] continent:map[code:EU geoname_id:6255148 names:map[de:Europa en:Europe es:Europa fr:Europe ja:ヨーロッパ pt-BR:Europa ru:Европа zh-CN:欧洲]] country:map[geoname_id:2635167 iso_code:GB names:map[de:Vereinigtes Königreich en:United Kingdom es:Reino Unido fr:Royaume-Uni ja:イギリス pt-BR:Reino Unido ru:Великобритания zh-CN:英国]] location:map[accuracy_radius:10 latitude:51.5142 longitude:-0.0931 time_zone:Europe/London] registered_country:map[geoname_id:6252001 iso_code:US names:map[de:USA en:United States es:Estados Unidos fr:États-Unis ja:アメリカ合衆国 pt-BR:Estados Unidos ru:США zh-CN:美国]] subdivisions:[map[geoname_id:6269131 iso_code:ENG names:map[en:England es:Inglaterra fr:Angleterre pt-BR:Inglaterra]]]]
5655
}
@@ -62,7 +61,7 @@ func ExampleReader_Networks() {
6261
iferr!=nil {
6362
log.Fatal(err)
6463
}
65-
deferdb.Close()//nolint:errcheck // error doesn't matter
64+
deferdb.Close()
6665

6766
forresult:=rangedb.Networks() {
6867
record:=struct {
@@ -108,7 +107,7 @@ func ExampleReader_Verify() {
108107
iferr!=nil {
109108
log.Fatal(err)
110109
}
111-
deferdb.Close()//nolint:errcheck // error doesn't matter
110+
deferdb.Close()
112111

113112
// Verify database integrity
114113
iferr:=db.Verify();err!=nil {
@@ -142,7 +141,7 @@ func ExampleReader_NetworksWithin() {
142141
iferr!=nil {
143142
log.Fatal(err)
144143
}
145-
deferdb.Close()//nolint:errcheck // error doesn't matter
144+
deferdb.Close()
146145

147146
prefix,err:=netip.ParsePrefix("1.0.0.0/8")
148147
iferr!=nil {
@@ -172,6 +171,80 @@ func ExampleReader_NetworksWithin() {
172171
// 1.0.128.0/17: Cable/DSL
173172
}
174173

174+
// This example demonstrates how to use SkipEmptyValues to iterate only over
175+
// networks that have actual data, skipping those with empty maps or arrays.
176+
funcExampleSkipEmptyValues() {
177+
db,err:=maxminddb.Open("test-data/test-data/GeoIP2-Anonymous-IP-Test.mmdb")
178+
iferr!=nil {
179+
log.Fatal(err)
180+
}
181+
deferdb.Close()
182+
183+
// Without SkipEmptyValues, you get all networks including empty ones
184+
fmt.Println("All networks:")
185+
count:=0
186+
forresult:=rangedb.Networks() {
187+
ifresult.Err()!=nil {
188+
log.Panic(result.Err())
189+
}
190+
count++
191+
ifcount>10 {
192+
fmt.Printf("... (%d more networks, many with empty data)\n",529-count)
193+
break
194+
}
195+
196+
varrecordmap[string]any
197+
err:=result.Decode(&record)
198+
iferr!=nil {
199+
log.Panic(err)
200+
}
201+
202+
iflen(record)==0 {
203+
fmt.Printf("%s: (empty)\n",result.Prefix())
204+
}else {
205+
fmt.Printf("%s: %v\n",result.Prefix(),record)
206+
}
207+
}
208+
209+
fmt.Println("\nOnly networks with data:")
210+
// With SkipEmptyValues, you only get networks with actual data
211+
forresult:=rangedb.Networks(maxminddb.SkipEmptyValues()) {
212+
ifresult.Err()!=nil {
213+
log.Panic(result.Err())
214+
}
215+
216+
varrecordmap[string]any
217+
err:=result.Decode(&record)
218+
iferr!=nil {
219+
log.Panic(result.Err())
220+
}
221+
fmt.Printf("%s: %v\n",result.Prefix(),record)
222+
}
223+
224+
// Output:
225+
// All networks:
226+
// 1.0.0.0/15: (empty)
227+
// 1.2.0.0/16: map[is_anonymous:true is_anonymous_vpn:true]
228+
// 1.3.0.0/16: (empty)
229+
// 1.4.0.0/14: (empty)
230+
// 1.8.0.0/13: (empty)
231+
// 1.16.0.0/12: (empty)
232+
// 1.32.0.0/11: (empty)
233+
// 1.64.0.0/11: (empty)
234+
// 1.96.0.0/12: (empty)
235+
// 1.112.0.0/13: (empty)
236+
// ... (518 more networks, many with empty data)
237+
//
238+
// Only networks with data:
239+
// 1.2.0.0/16: map[is_anonymous:true is_anonymous_vpn:true]
240+
// 1.124.213.1/32: map[is_anonymous:true is_anonymous_vpn:true is_tor_exit_node:true]
241+
// 65.0.0.0/13: map[is_anonymous:true is_tor_exit_node:true]
242+
// 71.160.223.0/24: map[is_anonymous:true is_hosting_provider:true]
243+
// 81.2.69.0/24: map[is_anonymous:true is_anonymous_vpn:true is_hosting_provider:true is_public_proxy:true is_residential_proxy:true is_tor_exit_node:true]
244+
// 186.30.236.0/24: map[is_anonymous:true is_public_proxy:true]
245+
// abcd:1000::/112: map[is_anonymous:true is_public_proxy:true]
246+
}
247+
175248
// CustomCity represents a simplified city record with custom unmarshaling.
176249
// This demonstrates the Unmarshaler interface for custom decoding.
177250
typeCustomCitystruct {
@@ -252,7 +325,7 @@ func ExampleUnmarshaler() {
252325
iferr!=nil {
253326
log.Fatal(err)
254327
}
255-
deferdb.Close()//nolint:errcheck // error doesn't matter
328+
deferdb.Close()
256329

257330
addr:=netip.MustParseAddr("81.2.69.142")
258331

‎internal/decoder/reflection.go‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,29 @@ func New(buffer []byte) ReflectionDecoder {
3030
}
3131
}
3232

33+
// IsEmptyValueAt checks if the value at the given offset is an empty map or array.
34+
// Returns true if the value is a map or array with size 0.
35+
func (d*ReflectionDecoder)IsEmptyValueAt(offsetuint) (bool,error) {
36+
dataOffset:=offset
37+
for {
38+
kindNum,size,newOffset,err:=d.decodeCtrlData(dataOffset)
39+
iferr!=nil {
40+
returnfalse,err
41+
}
42+
43+
ifkindNum==KindPointer {
44+
dataOffset,_,err=d.decodePointer(size,newOffset)
45+
iferr!=nil {
46+
returnfalse,err
47+
}
48+
continue
49+
}
50+
51+
// Check if it's a map or array with size 0
52+
return (kindNum==KindMap||kindNum==KindSlice)&&size==0,nil
53+
}
54+
}
55+
3356
// Decode decodes the data value at offset and stores it in the value
3457
// pointed at by v.
3558
func (d*ReflectionDecoder)Decode(offsetuint,vany)error {

‎traverse.go‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ type netNode struct {
1919
typenetworkOptionsstruct {
2020
includeAliasedNetworksbool
2121
includeEmptyNetworksbool
22+
skipEmptyValuesbool
2223
}
2324

2425
var (
@@ -46,6 +47,16 @@ func IncludeNetworksWithoutData() NetworksOption {
4647
}
4748
}
4849

50+
// SkipEmptyValues is an option for Networks and NetworksWithin that makes
51+
// them skip networks whose data is an empty map or empty array. This is
52+
// useful for databases that store empty maps or arrays for records without
53+
// meaningful data, allowing iteration over only records with actual content.
54+
funcSkipEmptyValues()NetworksOption {
55+
returnfunc(networks*networkOptions) {
56+
networks.skipEmptyValues=true
57+
}
58+
}
59+
4960
// Networks returns an iterator that can be used to traverse the networks in
5061
// the database.
5162
//
@@ -169,6 +180,17 @@ func (r *Reader) NetworksWithin(prefix netip.Prefix, options ...NetworksOption)
169180

170181
ifnode.pointer>r.Metadata.NodeCount {
171182
offset,err:=r.resolveDataPointer(node.pointer)
183+
184+
// Check if we should skip empty values (only if no error)
185+
iferr==nil&&n.skipEmptyValues {
186+
varisEmptybool
187+
isEmpty,err=r.decoder.IsEmptyValueAt(uint(offset))
188+
iferr==nil&&isEmpty {
189+
// Skip this empty value
190+
break
191+
}
192+
}
193+
172194
ok:=yield(Result{
173195
decoder:r.decoder,
174196
ip:mappedIP(node.ip),

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp