Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commit7596564

Browse files
authored
feat: Ability to Limit the Number of Scrapers Running Concurrently (#1664)
* Ability to Limit the Number of Scrapers Running Concurrently* Remove debugging log message
1 parenta21431a commit7596564

File tree

4 files changed

+43
-15
lines changed

4 files changed

+43
-15
lines changed

‎README.md

Lines changed: 15 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -136,17 +136,18 @@ left arrow - cycles backwards in gallery
136136
right arrow - cycles forward in gallery
137137
esc - closes details pane
138138

139-
####using Command Line Arguments
140-
| Command line parameter| Type| Description|
141-
|------------------------|-------|--------------
142-
|`--enableLocalStorage`| boolean|Use local folder to store application data|
143-
|`--app_dir`| String|path to the application directory|
144-
|`--cache_dir`| String|path to the tempoarary scraper cache directory|
145-
|`--imgproxy_dir`| String|path to the imageproxy directory|
146-
|`--search_dir`| String| path to the Search Index directory|
147-
|`--preview_dir`| String| path to the Scraper Cache directory|
148-
|`--scriptsheatmap_dir`| String| path to the scripts_heatmap directory|
149-
|`--myfiles_dir`| String| path to the myfiles directory for serving users own content (eg images|
150-
|`--databaseurl`| String|override default database path|
151-
|`--web_port`| Int| override default Web Page port 9999|
152-
|`--ws_addr`| String| override default Websocket address from the default 0.0.0.0:9998|
139+
####using Command Line Arguments/Environment Variables
140+
| Command line parameter| Environment Variable| Type| Description|
141+
|------------------------|--------------|------|-------------|
142+
|`--enableLocalStorage`|| boolean|Use local folder to store application data|
143+
|`--app_dir`| XBVR_APPDIR| String|path to the application directory|
144+
|`--cache_dir`| XBVR_CACHEDIR| String|path to the tempoarary scraper cache directory|
145+
|`--imgproxy_dir`| XBVR_IMAGEPROXYDIR| String|path to the imageproxy directory|
146+
|`--search_dir`| XBVR_SEARCHDIR| String| path to the Search Index directory|
147+
|`--preview_dir`| XBVR_VIDEOPREVIEWDIR| String| path to the Scraper Cache directory|
148+
|`--scriptsheatmap_dir`| XBVR_SCRIPTHEATMAPDIR| String| path to the scripts_heatmap directory|
149+
|`--myfiles_dir`| XBVR_MYFILESDIR| String| path to the myfiles directory for serving users own content (eg images|
150+
|`--databaseurl`| DATABASE_URL| String|override default database path|
151+
|`--web_port`| XBVR_WEB_PORT| Int| override default Web Page port 9999|
152+
|`--ws_addr`| DB_CONNECTION_POOL_SIZE| String| override default Websocket address from the default 0.0.0.0:9998|
153+
|`--concurrent_scrapers`| CONCURRENT_SCRAPERS| String| set the number of scrapers that run concurrently default 9999|

‎pkg/common/common.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ type EnvConfigSpec struct {
2222
WsAddrstring`envconfig:"XBVR_WS_ADDR" required:"false" default:""`
2323
WebPortint`envconfig:"XBVR_WEB_PORT" required:"false" default:"0"`
2424
DBConnectionPoolSizeint`envconfig:"DB_CONNECTION_POOL_SIZE" required:"false" default:"0"`
25+
ConcurrentScrapersint`envconfig:"CONCURRENT_SCRAPERS" required:"false" default:"9999"`
2526
}
2627

2728
varEnvConfigEnvConfigSpec

‎pkg/common/paths.go

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ var MyFilesDir string
2424
varDownloadDirstring
2525
varWebPortint
2626
varDBConnectionPoolSizeint
27+
varConcurrentScrapersint
2728

2829
funcDirSize(pathstring) (int64,error) {
2930
varsizeint64
@@ -53,6 +54,7 @@ func InitPaths() {
5354
web_port:=flag.Int("web_port",0,"Optional: override default Web Page port 9999")
5455
ws_addr:=flag.String("ws_addr","","Optional: override default Websocket address from the default 0.0.0.0:9998")
5556
db_connection_pool_size:=flag.Int("db_connection_pool_size",0,"Optional: sets a limit to the number of db connections while scraping")
57+
concurrentSscrapers:=flag.Int("concurrent_scrapers",0,"Optional: sets a limit to the number of concurrent scrapers")
5658

5759
flag.Parse()
5860

@@ -120,6 +122,11 @@ func InitPaths() {
120122
}else {
121123
DBConnectionPoolSize=EnvConfig.DBConnectionPoolSize
122124
}
125+
if*concurrentSscrapers!=0 {
126+
ConcurrentScrapers=*concurrentSscrapers
127+
}else {
128+
ConcurrentScrapers=EnvConfig.ConcurrentScrapers
129+
}
123130

124131
_=os.MkdirAll(AppDir,os.ModePerm)
125132
_=os.MkdirAll(ImgDir,os.ModePerm)

‎pkg/tasks/content.go

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -117,12 +117,31 @@ func runScrapers(knownScenes []string, toScrape string, updateSite bool, collect
117117

118118
varwg sync.WaitGroup
119119

120+
sitecnt:=1
121+
concurrent_scrapers:=common.ConcurrentScrapers
122+
ifconcurrent_scrapers==0 {
123+
concurrent_scrapers=99999
124+
}
120125
iflen(sites)>0 {
121126
for_,site:=rangesites {
122127
for_,scraper:=rangescrapers {
123128
ifsite.ID==scraper.ID {
124129
wg.Add(1)
125-
goscraper.Scrape(&wg,updateSite,knownScenes,collectedScenes,singleSceneURL,singeScrapeAdditionalInfo,site.LimitScraping)
130+
gofunc(scraper models.Scraper) {
131+
scraper.Scrape(&wg,updateSite,knownScenes,collectedScenes,singleSceneURL,singeScrapeAdditionalInfo,site.LimitScraping)
132+
varsite models.Site
133+
err:=site.GetIfExist(scraper.ID)
134+
iferr!=nil {
135+
log.Error(err)
136+
return
137+
}
138+
site.Save()
139+
}(scraper)
140+
141+
ifsitecnt%concurrent_scrapers==0 {// processing batches of 35 sites
142+
wg.Wait()
143+
}
144+
sitecnt++
126145
}
127146
}
128147
}

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp