Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commit7dc8b5a

Browse files
committed
Merge branch '647-slow-clone-list' into 'master'
fix: improve performance of the dblab clone list command (#647)Closes #647See merge request postgres-ai/database-lab!1054
2 parents6541194 +28c2e6d commit7dc8b5a

File tree

19 files changed

+559
-79
lines changed

19 files changed

+559
-79
lines changed

‎engine/.gitlab-ci.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
default:
22
image:
3-
name:golang:1.23
3+
name:golang:1.24
44
pull_policy:if-not-present
55

66
stages:
@@ -58,7 +58,7 @@ lint:
5858
build-binary-alpine:
5959
<<:*only_engine
6060
image:
61-
name:golang:1.23-alpine
61+
name:golang:1.24-alpine
6262
pull_policy:if-not-present
6363
stage:build-binary
6464
artifacts:

‎engine/Dockerfile.dblab-server-debug‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
# How to start a container: https://postgres.ai/docs/how-to-guides/administration/engine-manage
22

33
# Compile stage
4-
FROM golang:1.23 AS build-env
4+
FROM golang:1.24 AS build-env
55

66
# Build Delve
77
RUN go install github.com/go-delve/delve/cmd/dlv@latest

‎engine/Makefile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ help: ## Display the help message
3434
all: clean build## Build all binary components of the project
3535

3636
install-lint:## Install the linter to $GOPATH/bin which is expected to be in $PATH
37-
curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh| sh -s -- -b$(go env GOPATH)/bin v1.61.0
37+
curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh| sh -s -- -b$(go env GOPATH)/bin v1.64.8
3838

3939
run-lint:## Run linters
4040
golangci-lint run

‎engine/go.mod‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
modulegitlab.com/postgres-ai/database-lab/v3
22

3-
go1.23.12
3+
go1.24.7
44

55
require (
66
github.com/AlekSi/pointerv1.2.0

‎engine/go.sum‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -310,8 +310,6 @@ github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXP
310310
github.com/gogo/protobufv1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
311311
github.com/gogo/protobufv1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
312312
github.com/gogo/protobufv1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
313-
github.com/golang-jwt/jwt/v4v4.5.1 h1:JdqV9zKUdtaa9gdPlywC3aeoEsR681PlKC+4F5gQgeo=
314-
github.com/golang-jwt/jwt/v4v4.5.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
315313
github.com/golang-jwt/jwt/v4v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXeUI=
316314
github.com/golang-jwt/jwt/v4v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
317315
github.com/golang/glogv0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=

‎engine/internal/cloning/base.go‎

Lines changed: 23 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -435,8 +435,10 @@ func (c *Base) refreshCloneMetadata(w *CloneWrapper) {
435435
return
436436
}
437437

438+
c.cloneMutex.Lock()
438439
w.Clone.Metadata.CloneDiffSize=sessionState.CloneDiffSize
439440
w.Clone.Metadata.LogicalSize=sessionState.LogicalReferenced
441+
c.cloneMutex.Unlock()
440442
}
441443

442444
// UpdateClone updates clone.
@@ -527,11 +529,6 @@ func (c *Base) ResetClone(cloneID string, resetOptions types.ResetCloneRequest)
527529
log.Warn("clone has dependent snapshots",cloneID)
528530
c.cloneMutex.Lock()
529531
w.Clone.Revision++
530-
w.Clone.HasDependent=true
531-
c.cloneMutex.Unlock()
532-
}else {
533-
c.cloneMutex.Lock()
534-
w.Clone.HasDependent=false
535532
c.cloneMutex.Unlock()
536533
}
537534

@@ -630,6 +627,8 @@ func (c *Base) GetClones() []*models.Clone {
630627
clones:=make([]*models.Clone,0,c.lenClones())
631628

632629
c.cloneMutex.RLock()
630+
requestsByPool:=make(map[string][]resources.SessionStateRequest)
631+
633632
for_,cloneWrapper:=rangec.clones {
634633
ifcloneWrapper.Clone.Snapshot!=nil {
635634
snapshot,err:=c.getSnapshotByID(cloneWrapper.Clone.Snapshot.ID)
@@ -642,12 +641,30 @@ func (c *Base) GetClones() []*models.Clone {
642641
}
643642
}
644643

645-
c.refreshCloneMetadata(cloneWrapper)
644+
ifcloneWrapper.Session!=nil&&cloneWrapper.Clone!=nil {
645+
pool:=cloneWrapper.Session.Pool
646+
requestsByPool[pool]=append(requestsByPool[pool], resources.SessionStateRequest{
647+
CloneID:cloneWrapper.Clone.ID,
648+
Branch:cloneWrapper.Clone.Branch,
649+
})
650+
}
646651

647652
clones=append(clones,cloneWrapper.Clone)
648653
}
649654
c.cloneMutex.RUnlock()
650655

656+
sessionStates,err:=c.provision.GetBatchSessionState(requestsByPool)
657+
iferr!=nil {
658+
log.Err("failed to get batch session states: ",err)
659+
}
660+
661+
for_,clone:=rangeclones {
662+
ifstate,ok:=sessionStates[clone.ID];ok {
663+
clone.Metadata.CloneDiffSize=state.CloneDiffSize
664+
clone.Metadata.LogicalSize=state.LogicalReferenced
665+
}
666+
}
667+
651668
sort.Slice(clones,func(i,jint)bool {
652669
returnclones[i].CreatedAt.After(clones[j].CreatedAt.Time)
653670
})

‎engine/internal/provision/mode_local.go‎

Lines changed: 37 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -274,10 +274,8 @@ func (p *Provisioner) ResetSession(session *resources.Session, clone *models.Clo
274274
returnnil,errors.Wrap(err,"failed to stop container")
275275
}
276276

277-
ifclone.Revision==branching.DefaultRevision||!clone.HasDependent {
278-
iferr=fsm.DestroyClone(clone.Branch,name,clone.Revision);err!=nil {
279-
returnnil,errors.Wrap(err,"failed to destroy clone")
280-
}
277+
iferr=fsm.DestroyClone(clone.Branch,name,clone.Revision);err!=nil {
278+
returnnil,errors.Wrap(err,"failed to destroy clone")
281279
}
282280

283281
iferr=newFSManager.CreateClone(clone.Branch,name,snapshot.ID,clone.Revision);err!=nil {
@@ -300,9 +298,14 @@ func (p *Provisioner) ResetSession(session *resources.Session, clone *models.Clo
300298
}
301299

302300
snapshotModel:=&models.Snapshot{
303-
ID:snapshot.ID,
304-
CreatedAt:models.NewLocalTime(snapshot.CreatedAt),
305-
DataStateAt:models.NewLocalTime(snapshot.DataStateAt),
301+
ID:snapshot.ID,
302+
CreatedAt:models.NewLocalTime(snapshot.CreatedAt),
303+
DataStateAt:models.NewLocalTime(snapshot.DataStateAt),
304+
PhysicalSize:snapshot.Used,
305+
LogicalSize:snapshot.LogicalReferenced,
306+
Pool:snapshot.Pool,
307+
Branch:snapshot.Branch,
308+
Message:snapshot.Message,
306309
}
307310

308311
returnsnapshotModel,nil
@@ -335,6 +338,31 @@ func (p *Provisioner) GetSessionState(s *resources.Session, branch, cloneID stri
335338
returnfsm.GetSessionState(branch,cloneID)
336339
}
337340

341+
// GetBatchSessionState retrieves session states for multiple clones efficiently.
342+
func (p*Provisioner)GetBatchSessionState(batchmap[string][]resources.SessionStateRequest) (map[string]resources.SessionState,error) {
343+
batchResults:=make(map[string]resources.SessionState)
344+
345+
forpoolName,reqs:=rangebatch {
346+
fsm,err:=p.pm.GetFSManager(poolName)
347+
iferr!=nil {
348+
log.Err(fmt.Sprintf("failed to find filesystem manager for pool %s: %v",poolName,err))
349+
continue
350+
}
351+
352+
results,err:=fsm.GetBatchSessionState(reqs)
353+
iferr!=nil {
354+
log.Err(fmt.Sprintf("failed to get batch session state for pool %s: %v",poolName,err))
355+
continue
356+
}
357+
358+
forcloneID,state:=rangeresults {
359+
batchResults[cloneID]=state
360+
}
361+
}
362+
363+
returnbatchResults,nil
364+
}
365+
338366
// GetPoolEntryList provides an ordered list of available pools.
339367
func (p*Provisioner)GetPoolEntryList() []models.PoolEntry {
340368
fsmList:=p.pm.GetFSManagerOrderedList()
@@ -604,10 +632,8 @@ func (p *Provisioner) CleanupCloneDataset(clone *models.Clone, pool string) erro
604632
returnnil
605633
}
606634

607-
ifclone.Revision==branching.DefaultRevision&&!clone.HasDependent {
608-
iferr:=fsm.DestroyDataset(branching.CloneDataset(pool,clone.Branch,clone.ID));err!=nil {
609-
returnfmt.Errorf("failed to destroy clone dataset: %w",err)
610-
}
635+
iferr=fsm.DestroyClone(clone.Branch,clone.ID,clone.Revision);err!=nil {
636+
returnfmt.Errorf("failed to destroy clone: %w",err)
611637
}
612638

613639
returnnil

‎engine/internal/provision/mode_local_test.go‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,10 @@ func (m mockFSManager) GetSessionState(_, _ string) (*resources.SessionState, er
102102
returnnil,nil
103103
}
104104

105+
func (mmockFSManager)GetBatchSessionState(_ []resources.SessionStateRequest) (map[string]resources.SessionState,error) {
106+
returnmake(map[string]resources.SessionState),nil
107+
}
108+
105109
func (mmockFSManager)GetFilesystemState() (models.FileSystem,error) {
106110
return models.FileSystem{Mode:"zfs"},nil
107111
}
@@ -214,6 +218,14 @@ func (m mockFSManager) KeepRelation(_ string) error {
214218
returnnil
215219
}
216220

221+
func (mmockFSManager)GetDatasetOrigins(_string) []string {
222+
returnnil
223+
}
224+
225+
func (mmockFSManager)GetActiveDatasets(_string) ([]string,error) {
226+
returnnil,nil
227+
}
228+
217229
funcTestBuildPoolEntry(t*testing.T) {
218230
testCases:= []struct {
219231
pool*resources.Pool

‎engine/internal/provision/pool/manager.go‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ type Cloner interface {
3939
// StateReporter describes methods of state reporting.
4040
typeStateReporterinterface {
4141
GetSessionState(branch,namestring) (*resources.SessionState,error)
42+
GetBatchSessionState(requests []resources.SessionStateRequest) (map[string]resources.SessionState,error)
4243
GetFilesystemState() (models.FileSystem,error)
4344
}
4445

@@ -57,7 +58,7 @@ type Branching interface {
5758
VerifyBranchMetadata()error
5859
CreateDataset(datasetNamestring)error
5960
CreateBranch(branchName,snapshotIDstring)error
60-
DestroyDataset(branchNamestring) (errerror)
61+
DestroyDataset(datasetstring) (errerror)
6162
ListBranches() (map[string]string,error)
6263
ListAllBranches(filterPools []string) ([]models.BranchEntity,error)
6364
GetRepo() (*models.Repo,error)
@@ -78,6 +79,8 @@ type Branching interface {
7879
Reset(snapshotIDstring,options thinclones.ResetOptions)error
7980
HasDependentEntity(snapshotNamestring) ([]string,error)
8081
KeepRelation(snapshotNamestring)error
82+
GetDatasetOrigins(snapshotNamestring) []string
83+
GetActiveDatasets(datasetstring) ([]string,error)
8184
}
8285

8386
// Pooler describes methods for Pool providing.

‎engine/internal/provision/resources/resources.go‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,3 +48,9 @@ type SessionState struct {
4848
CloneDiffSizeuint64
4949
LogicalReferenceduint64
5050
}
51+
52+
// SessionStateRequest defines a request for batch session state retrieval.
53+
typeSessionStateRequeststruct {
54+
CloneIDstring
55+
Branchstring
56+
}

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp