Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commit9711d20

Browse files
toshskicrwxaj
andauthored
scrape: add StashDB scraper (#1894)
* Scrape Stashdb studios* Fix waitgroup change* Save duration, url and tags from stashdb* Fix missing fields when actor created---------Co-authored-by: crwxaj <52156245+crwxaj@users.noreply.github.com>
1 parent74bd2c0 commit9711d20

File tree

10 files changed

+369
-27
lines changed

10 files changed

+369
-27
lines changed

‎pkg/api/options.go

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1017,6 +1017,7 @@ func (i ConfigResource) createCustomSite(req *restful.Request, resp *restful.Res
10171017
scrapers:=make(map[string][]config.ScraperConfig)
10181018
scrapers["povr"]=scraperConfig.CustomScrapers.PovrScrapers
10191019
scrapers["slr"]=scraperConfig.CustomScrapers.SlrScrapers
1020+
scrapers["stashdb"]=scraperConfig.CustomScrapers.StashDbScrapers
10201021
scrapers["vrphub"]=scraperConfig.CustomScrapers.VrphubScrapers
10211022
scrapers["vrporn"]=scraperConfig.CustomScrapers.VrpornScrapers
10221023

@@ -1040,6 +1041,8 @@ func (i ConfigResource) createCustomSite(req *restful.Request, resp *restful.Res
10401041
scrapers["povr"]=append(scrapers["povr"],scraper)
10411042
case"sexlikereal":
10421043
scrapers["slr"]=append(scrapers["slr"],scraper)
1044+
case"stashdb":
1045+
scrapers["stashdb"]=append(scrapers["stashdb"],scraper)
10431046
case"vrphub":
10441047
scrapers["vrphub"]=append(scrapers["vrphub"],scraper)
10451048
case"vrporn":
@@ -1048,6 +1051,7 @@ func (i ConfigResource) createCustomSite(req *restful.Request, resp *restful.Res
10481051
}
10491052
scraperConfig.CustomScrapers.PovrScrapers=scrapers["povr"]
10501053
scraperConfig.CustomScrapers.SlrScrapers=scrapers["slr"]
1054+
scraperConfig.CustomScrapers.StashDbScrapers=scrapers["stashdb"]
10511055
scraperConfig.CustomScrapers.VrphubScrapers=scrapers["vrphub"]
10521056
scraperConfig.CustomScrapers.VrpornScrapers=scrapers["vrporn"]
10531057
fName:=filepath.Join(common.AppDir,"scrapers.json")

‎pkg/config/scraper_list.go

Lines changed: 13 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -21,16 +21,18 @@ type ScraperList struct {
2121
XbvrScrapersXbvrScrapers`json:"xbvr"`
2222
}
2323
typeXbvrScrapersstruct {
24-
PovrScrapers []ScraperConfig`json:"povr"`
25-
SlrScrapers []ScraperConfig`json:"slr"`
26-
VrpornScrapers []ScraperConfig`json:"vrporn"`
27-
VrphubScrapers []ScraperConfig`json:"vrphub"`
24+
PovrScrapers []ScraperConfig`json:"povr"`
25+
SlrScrapers []ScraperConfig`json:"slr"`
26+
StashDbScrapers []ScraperConfig`json:"stashdb"`
27+
VrpornScrapers []ScraperConfig`json:"vrporn"`
28+
VrphubScrapers []ScraperConfig`json:"vrphub"`
2829
}
2930
typeCustomScrapersstruct {
30-
PovrScrapers []ScraperConfig`json:"povr"`
31-
SlrScrapers []ScraperConfig`json:"slr"`
32-
VrpornScrapers []ScraperConfig`json:"vrporn"`
33-
VrphubScrapers []ScraperConfig`json:"vrphub"`
31+
PovrScrapers []ScraperConfig`json:"povr"`
32+
SlrScrapers []ScraperConfig`json:"slr"`
33+
StashDbScrapers []ScraperConfig`json:"stashdb"`
34+
VrpornScrapers []ScraperConfig`json:"vrporn"`
35+
VrphubScrapers []ScraperConfig`json:"vrphub"`
3436
}
3537
typeScraperConfigstruct {
3638
IDstring`json:"-"`
@@ -71,16 +73,19 @@ func (o *ScraperList) Load() error {
7173

7274
SetSiteId(&o.XbvrScrapers.PovrScrapers,"")
7375
SetSiteId(&o.XbvrScrapers.SlrScrapers,"")
76+
SetSiteId(&o.XbvrScrapers.StashDbScrapers,"")
7477
SetSiteId(&o.XbvrScrapers.VrphubScrapers,"")
7578
SetSiteId(&o.XbvrScrapers.VrpornScrapers,"")
7679
SetSiteId(&o.CustomScrapers.PovrScrapers,"povr")
7780
SetSiteId(&o.CustomScrapers.SlrScrapers,"slr")
81+
SetSiteId(&o.CustomScrapers.StashDbScrapers,"stashdb")
7882
SetSiteId(&o.CustomScrapers.VrphubScrapers,"vrphub")
7983
SetSiteId(&o.CustomScrapers.VrpornScrapers,"vrporn")
8084

8185
// remove custom sites that are now offical for the same aggregation site
8286
o.CustomScrapers.PovrScrapers=RemoveCustomListNowOffical(o.CustomScrapers.PovrScrapers,o.XbvrScrapers.PovrScrapers)
8387
o.CustomScrapers.SlrScrapers=RemoveCustomListNowOffical(o.CustomScrapers.SlrScrapers,o.XbvrScrapers.SlrScrapers)
88+
o.CustomScrapers.StashDbScrapers=RemoveCustomListNowOffical(o.CustomScrapers.StashDbScrapers,o.XbvrScrapers.StashDbScrapers)
8489
o.CustomScrapers.VrphubScrapers=RemoveCustomListNowOffical(o.CustomScrapers.VrphubScrapers,o.XbvrScrapers.VrphubScrapers)
8590
o.CustomScrapers.VrpornScrapers=RemoveCustomListNowOffical(o.CustomScrapers.VrpornScrapers,o.XbvrScrapers.VrpornScrapers)
8691

‎pkg/externalreference/stashdb.go

Lines changed: 31 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -91,22 +91,39 @@ func matchOnSceneUrl() {
9191
varxbvrIduint
9292
varxbvrSceneIdstring
9393

94+
for_,unmatchedXbvrScene:=rangeunmatchedXbvrScenes {
95+
stashId:=strings.TrimPrefix(unmatchedXbvrScene.SceneID,"stash-")
96+
ifstashId==stashScene.ExternalId {
97+
// xbvrLink := models.ExternalReferenceLink{InternalTable: "scenes", InternalDbId: scene.ID, InternalNameId: scene.SceneID,
98+
// ExternalReferenceID: stashScene.ID, ExternalSource: stashScene.ExternalSource, ExternalId: stashScene.ExternalId, MatchType: 20}
99+
//stashScene.XbvrLinks = append(stashScene.XbvrLinks, xbvrLink)
100+
//stashScene.Save()
101+
//var externalData models.StashScene
102+
//json.Unmarshal([]byte(stashScene.ExternalData), &externalData)
103+
//matchPerformerName(externalData, scene, 20)
104+
xbvrId=unmatchedXbvrScene.ID
105+
xbvrSceneId=unmatchedXbvrScene.SceneID
106+
}
107+
}
108+
94109
// see if we can link to an xbvr scene based on the urls
95-
for_,url:=rangescene.URLs {
96-
ifurl.Type=="STUDIO" {
97-
varxbvrScene models.Scene
98-
for_,scene:=rangeunmatchedXbvrScenes {
99-
sceneurl:=removeQueryFromURL(scene.SceneURL)
100-
tmpurl:=removeQueryFromURL(url.URL)
101-
sceneurl=simplifyUrl(sceneurl)
102-
tmpurl=simplifyUrl(tmpurl)
103-
ifstrings.EqualFold(sceneurl,tmpurl) {
104-
xbvrScene=scene
110+
ifxbvrId==0 {
111+
for_,url:=rangescene.URLs {
112+
ifurl.Type=="STUDIO" {
113+
varxbvrScene models.Scene
114+
for_,scene:=rangeunmatchedXbvrScenes {
115+
sceneurl:=removeQueryFromURL(scene.SceneURL)
116+
tmpurl:=removeQueryFromURL(url.URL)
117+
sceneurl=simplifyUrl(sceneurl)
118+
tmpurl=simplifyUrl(tmpurl)
119+
ifstrings.EqualFold(sceneurl,tmpurl) {
120+
xbvrScene=scene
121+
}
122+
}
123+
ifxbvrScene.ID!=0 {
124+
xbvrId=xbvrScene.ID
125+
xbvrSceneId=xbvrScene.SceneID
105126
}
106-
}
107-
ifxbvrScene.ID!=0 {
108-
xbvrId=xbvrScene.ID
109-
xbvrSceneId=xbvrScene.SceneID
110127
}
111128
}
112129
}

‎pkg/models/model_scene.go

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -462,8 +462,12 @@ func SceneCreateUpdateFromExternal(db *gorm.DB, ext ScrapedScene) error {
462462
}
463463
}
464464
ifext.ActorDetails[name].ProfileUrl!="" {
465-
iftmpActor.AddToActorUrlArray(ActorLink{Url:ext.ActorDetails[name].ProfileUrl,Type:ext.ActorDetails[name].Source}) {
466-
saveActor=true
465+
ifstrings.HasPrefix(ext.ActorDetails[name].ProfileUrl,"https://stashdb.org/performers/") {
466+
467+
}else {
468+
iftmpActor.AddToActorUrlArray(ActorLink{Url:ext.ActorDetails[name].ProfileUrl,Type:ext.ActorDetails[name].Source}) {
469+
saveActor=true
470+
}
467471
}
468472
}
469473
ifsaveActor {

‎pkg/models/model_scraper.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@ type ActorDetails struct {
5454
ImageUrlstring
5555
ProfileUrlstring
5656
Sourcestring
57+
StashDatastring
5758
}
5859
typeTrailerScrapestruct {
5960
SceneUrlstring`json:"scene_url"`// url of the page to be scrapped

‎pkg/models/model_stashdb.go

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -73,14 +73,25 @@ type StashScene struct {
7373
StudioStashStudio`json:"studio"`
7474
Durationint`json:"duration"`
7575
Codestring`json:"code"`
76-
Images []Image`json:"images"`
76+
Images []StashImages`json:"images"`
77+
Tags []StashTags`json:"tags"`
7778
}
7879

7980
typeStashPerformerAsstruct {
8081
PerformerStashPerformer``
8182
Asstring``
8283
}
8384

85+
typeStashImagesstruct {
86+
URLstring`json:"url"`
87+
Widthstring`json:"width"`
88+
HeightSite`json:"height"`
89+
}
90+
typeStashTagsstruct {
91+
IDstring`json:"id"`
92+
Namestring`json:"name"`
93+
}
94+
8495
typeDELETEStashPerformerMinstruct {
8596
IDstring`json:"id"`
8697
Updatedstring`json:"updated"`

‎pkg/scrape/stashdb.go

Lines changed: 145 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,12 @@ type QueryPerformerResultType struct {
7676
Countint`json:"count"`
7777
Performers []models.StashPerformer`json:"performers"`
7878
}
79+
typeFindSceneResultstruct {
80+
DataFindSceneData`json:"data"`
81+
}
82+
typeFindSceneDatastruct {
83+
Scene models.StashScene`json:"findScene"`
84+
}
7985
typeFindPerformerScenesDatastruct {
8086
Performer models.StashPerformer`json:"findPerformer"`
8187
}
@@ -135,6 +141,11 @@ fingerprints{
135141
submissions
136142
}
137143
duration
144+
tags {
145+
id
146+
name
147+
}
148+
=======
138149
code
139150
deleted
140151
`
@@ -379,7 +390,65 @@ func GetScenePage(variables string) QueryScenesResult {
379390
queryScenes(input: $input) {
380391
count
381392
scenes{
382-
`+sceneFieldList+`
393+
id
394+
title
395+
details
396+
release_date
397+
date
398+
updated
399+
urls{
400+
url
401+
type
402+
site {
403+
id
404+
name
405+
description
406+
url
407+
regex
408+
valid_types
409+
}
410+
}
411+
studio{
412+
id
413+
name
414+
updated
415+
parent {
416+
id
417+
name
418+
}
419+
}
420+
images{
421+
url
422+
width
423+
height
424+
}
425+
performers{
426+
performer{
427+
id
428+
updated
429+
gender
430+
name
431+
aliases
432+
images{
433+
url
434+
width
435+
height
436+
}
437+
}
438+
as
439+
}
440+
fingerprints{
441+
hash
442+
duration
443+
submissions
444+
}
445+
tags {
446+
id
447+
name
448+
}
449+
duration
450+
code
451+
deleted
383452
}
384453
}
385454
}
@@ -392,6 +461,81 @@ func GetScenePage(variables string) QueryScenesResult {
392461
returndata
393462
}
394463

464+
funcGetSceneFromStash(sceneIdstring) models.StashScene {
465+
variables:=`{"id": "`+sceneId+`"} `
466+
query:=`
467+
query findScene($id: ID!) {
468+
findScene(id: $id) {
469+
id
470+
title
471+
details
472+
release_date
473+
date
474+
updated
475+
urls{
476+
url
477+
type
478+
site {
479+
id
480+
name
481+
description
482+
url
483+
regex
484+
valid_types
485+
}
486+
}
487+
studio{
488+
id
489+
name
490+
updated
491+
parent {
492+
id
493+
name
494+
}
495+
}
496+
images{
497+
url
498+
width
499+
height
500+
}
501+
performers{
502+
performer{
503+
id
504+
updated
505+
gender
506+
name
507+
aliases
508+
images{
509+
url
510+
width
511+
height
512+
}
513+
}
514+
as
515+
}
516+
fingerprints{
517+
hash
518+
duration
519+
submissions
520+
}
521+
tags {
522+
id
523+
name
524+
}
525+
duration
526+
code
527+
deleted
528+
}
529+
}
530+
`
531+
532+
// Define the variables needed for your query as a Go map
533+
resp:=CallStashDb(query,variables)
534+
vardataFindSceneResult
535+
json.Unmarshal(resp,&data)
536+
returndata.Data.Scene
537+
}
538+
395539
funcsaveScenesToExternalReferences(scenesQueryScenesResult,studioIdstring) {
396540
tlog:=log.WithField("task","scrape")
397541
startTime:=time.Now()

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp