Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commit0aae3b1

Browse files
authored
scraper: Add RealVR Scraper (#1899)
* Inital CommitAppears to fully work. Some where in the code the part of the studio url is being added to the name used for the actor scraper. I can't seem to find this* Actor scraper and Site ID bug fix* Go fmt* Merge Typos
1 parentf6b19f9 commit0aae3b1

File tree

11 files changed

+103
-16
lines changed

11 files changed

+103
-16
lines changed

‎pkg/api/options.go

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1020,6 +1020,7 @@ func (i ConfigResource) createCustomSite(req *restful.Request, resp *restful.Res
10201020
scrapers["stashdb"]=scraperConfig.CustomScrapers.StashDbScrapers
10211021
scrapers["vrphub"]=scraperConfig.CustomScrapers.VrphubScrapers
10221022
scrapers["vrporn"]=scraperConfig.CustomScrapers.VrpornScrapers
1023+
scrapers["realvr"]=scraperConfig.CustomScrapers.RealVRScrapers
10231024

10241025
exists:=false
10251026
forkey,group:=rangescrapers {
@@ -1047,13 +1048,16 @@ func (i ConfigResource) createCustomSite(req *restful.Request, resp *restful.Res
10471048
scrapers["vrphub"]=append(scrapers["vrphub"],scraper)
10481049
case"vrporn":
10491050
scrapers["vrporn"]=append(scrapers["vrporn"],scraper)
1051+
case"realvr":
1052+
scrapers["realvr"]=append(scrapers["realvr"],scraper)
10501053
}
10511054
}
10521055
scraperConfig.CustomScrapers.PovrScrapers=scrapers["povr"]
10531056
scraperConfig.CustomScrapers.SlrScrapers=scrapers["slr"]
10541057
scraperConfig.CustomScrapers.StashDbScrapers=scrapers["stashdb"]
10551058
scraperConfig.CustomScrapers.VrphubScrapers=scrapers["vrphub"]
10561059
scraperConfig.CustomScrapers.VrpornScrapers=scrapers["vrporn"]
1060+
scraperConfig.CustomScrapers.RealVRScrapers=scrapers["realvr"]
10571061
fName:=filepath.Join(common.AppDir,"scrapers.json")
10581062
list,_:=json.MarshalIndent(scraperConfig,""," ")
10591063
os.WriteFile(fName,list,0644)

‎pkg/api/scenes.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -382,6 +382,7 @@ func (i SceneResource) getFilters(req *restful.Request, resp *restful.Response)
382382
outAttributes=append(outAttributes,"Has Image")
383383
outAttributes=append(outAttributes,"VRPHub Scraper")
384384
outAttributes=append(outAttributes,"VRPorn Scraper")
385+
outAttributes=append(outAttributes,"RealVR Scraper")
385386
outAttributes=append(outAttributes,"Stashdb Linked")
386387
outAttributes=append(outAttributes,"Has Script Download")
387388
outAttributes=append(outAttributes,"Has AI Generated Script")
@@ -391,6 +392,7 @@ func (i SceneResource) getFilters(req *restful.Request, resp *restful.Response)
391392
outAttributes=append(outAttributes,"Available from Alternate Sites")
392393
outAttributes=append(outAttributes,"Available from POVR")
393394
outAttributes=append(outAttributes,"Available from VRPorn")
395+
outAttributes=append(outAttributes,"Available from RealVR")
394396
outAttributes=append(outAttributes,"Available from SLR")
395397
outAttributes=append(outAttributes,"Multiple Scenes Available at an Alternate Site")
396398
typeResultsstruct {

‎pkg/config/scraper_list.go

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,13 +24,15 @@ type XbvrScrapers struct {
2424
PovrScrapers []ScraperConfig`json:"povr"`
2525
SlrScrapers []ScraperConfig`json:"slr"`
2626
StashDbScrapers []ScraperConfig`json:"stashdb"`
27+
RealVRScrapers []ScraperConfig`json:"realvr"`
2728
VrpornScrapers []ScraperConfig`json:"vrporn"`
2829
VrphubScrapers []ScraperConfig`json:"vrphub"`
2930
}
3031
typeCustomScrapersstruct {
3132
PovrScrapers []ScraperConfig`json:"povr"`
3233
SlrScrapers []ScraperConfig`json:"slr"`
3334
StashDbScrapers []ScraperConfig`json:"stashdb"`
35+
RealVRScrapers []ScraperConfig`json:"realvr"`
3436
VrpornScrapers []ScraperConfig`json:"vrporn"`
3537
VrphubScrapers []ScraperConfig`json:"vrphub"`
3638
}
@@ -76,18 +78,21 @@ func (o *ScraperList) Load() error {
7678
SetSiteId(&o.XbvrScrapers.StashDbScrapers,"")
7779
SetSiteId(&o.XbvrScrapers.VrphubScrapers,"")
7880
SetSiteId(&o.XbvrScrapers.VrpornScrapers,"")
81+
SetSiteId(&o.XbvrScrapers.RealVRScrapers,"")
7982
SetSiteId(&o.CustomScrapers.PovrScrapers,"povr")
8083
SetSiteId(&o.CustomScrapers.SlrScrapers,"slr")
8184
SetSiteId(&o.CustomScrapers.StashDbScrapers,"stashdb")
8285
SetSiteId(&o.CustomScrapers.VrphubScrapers,"vrphub")
8386
SetSiteId(&o.CustomScrapers.VrpornScrapers,"vrporn")
87+
SetSiteId(&o.CustomScrapers.RealVRScrapers,"realvr")
8488

8589
// remove custom sites that are now offical for the same aggregation site
8690
o.CustomScrapers.PovrScrapers=RemoveCustomListNowOffical(o.CustomScrapers.PovrScrapers,o.XbvrScrapers.PovrScrapers)
8791
o.CustomScrapers.SlrScrapers=RemoveCustomListNowOffical(o.CustomScrapers.SlrScrapers,o.XbvrScrapers.SlrScrapers)
8892
o.CustomScrapers.StashDbScrapers=RemoveCustomListNowOffical(o.CustomScrapers.StashDbScrapers,o.XbvrScrapers.StashDbScrapers)
8993
o.CustomScrapers.VrphubScrapers=RemoveCustomListNowOffical(o.CustomScrapers.VrphubScrapers,o.XbvrScrapers.VrphubScrapers)
9094
o.CustomScrapers.VrpornScrapers=RemoveCustomListNowOffical(o.CustomScrapers.VrpornScrapers,o.XbvrScrapers.VrpornScrapers)
95+
o.CustomScrapers.RealVRScrapers=RemoveCustomListNowOffical(o.CustomScrapers.RealVRScrapers,o.XbvrScrapers.RealVRScrapers)
9196

9297
list,err:=json.MarshalIndent(o,""," ")
9398
iferr==nil {
@@ -149,8 +154,12 @@ func CheckMatchingSiteID(findSite ScraperConfig, searchList []ScraperConfig) boo
149154
funcSetSiteId(configList*[]ScraperConfig,customIdstring) {
150155
foridx,siteconfig:=range*configList {
151156
ifsiteconfig.FileID==""||customId!="" {
152-
id:=strings.TrimRight(siteconfig.URL,"/")
153-
siteconfig.ID=strings.ToLower(id[strings.LastIndex(id,"/")+1:])
157+
temp:=strings.TrimRight(siteconfig.URL,"/")
158+
id:=temp[strings.LastIndex(temp,"/")+1:]
159+
ifcustomId=="realvr" {
160+
id=id[:strings.Index(id,"-")-1]
161+
}
162+
siteconfig.ID=strings.ToLower(id)
154163
}else {
155164
siteconfig.ID=strings.ToLower(siteconfig.FileID)
156165
}

‎pkg/config/scrapers.json

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,8 @@
1313
"povr": [],
1414
"slr": [],
1515
"vrporn": [],
16-
"vrphub": []
16+
"vrphub": [],
17+
"realvr": []
1718
},
1819
"xbvr": {
1920
"povr": [

‎pkg/models/model_external_reference.go

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -306,7 +306,10 @@ func (o *ExternalReference) DetermineActorScraperBySiteId(siteId string) string
306306
return"vrphub scrape"
307307
}
308308
ifstrings.HasSuffix(site.Name,"VRPorn)") {
309-
return"slr scrape"
309+
return"vrporn scrape"
310+
}
311+
ifstrings.HasSuffix(site.Name,"RealVR)") {
312+
return"realvr scrape"
310313
}
311314
returnsiteId+" scrape"
312315
}
@@ -717,6 +720,8 @@ func (scrapeRules ActorScraperConfig) buildGenericActorScraperRules() {
717720
scrapeRules.GenericActorScrapingConfig["vrcosplayx scrape"]=siteDetails
718721
siteDetails.Domain="18vr.com"
719722
scrapeRules.GenericActorScrapingConfig["18vr scrape"]=siteDetails
723+
siteDetails.Domain="realvr.com"
724+
scrapeRules.GenericActorScrapingConfig["realvr scrape"]=siteDetails
720725

721726
siteDetails=GenericScraperRuleSet{}
722727
siteDetails.Domain="darkroomvr.com"

‎pkg/models/model_scene.go

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -895,6 +895,8 @@ func queryScenes(db *gorm.DB, r RequestSceneList) (*gorm.DB, *gorm.DB) {
895895
where=`scenes.scene_id like "vrphub-%"`
896896
case"VRPorn Scraper":
897897
where=`scenes.scene_id like "vrporn-%"`
898+
case"RealVR Scraper":
899+
where=`scenes.scene_id like "realvr-%"`
898900
case"Has Script Download":
899901
// querying the scenes in from alternate sources (stored in external_reference) has a performance impact, so it's user choice
900902
ifconfig.Advanced.UseAltSrcInFileMatching {
@@ -924,6 +926,8 @@ func queryScenes(db *gorm.DB, r RequestSceneList) (*gorm.DB, *gorm.DB) {
924926
where="exists (select 1 from external_reference_links where external_source like 'alternate scene %' and external_id like 'povr-%' and internal_db_id = scenes.id)"
925927
case"Available from VRPorn":
926928
where="exists (select 1 from external_reference_links where external_source like 'alternate scene %' and external_id like 'vrporn-%' and internal_db_id = scenes.id)"
929+
case"Available from RealVR":
930+
where="exists (select 1 from external_reference_links where external_source like 'alternate scene %' and external_id like 'realvr-%' and internal_db_id = scenes.id)"
927931
case"Available from SLR":
928932
where="exists (select 1 from external_reference_links where external_source like 'alternate scene %' and external_id like 'slr-%' and internal_db_id = scenes.id)"
929933
case"Available from Alternate Sites":

‎pkg/scrape/badoink.go

Lines changed: 18 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,6 @@ import (
1212

1313
"github.com/go-resty/resty/v2"
1414
"github.com/gocolly/colly/v2"
15-
"github.com/mozillazg/go-slugify"
1615
"github.com/nleeper/goment"
1716
"github.com/thoas/go-funk"
1817
"github.com/xbapps/xbvr/pkg/config"
@@ -22,12 +21,13 @@ import (
2221
"github.com/xbapps/xbvr/pkg/models"
2322
)
2423

25-
funcBadoinkSite(wg*models.ScrapeWG,updateSitebool,knownScenes []string,outchan<- models.ScrapedScene,singleSceneURLstring,scraperIDstring,siteIDstring,URLstring,singeScrapeAdditionalInfostring,limitScrapingbool)error {
24+
funcBadoinkSite(wg*models.ScrapeWG,updateSitebool,knownScenes []string,outchan<- models.ScrapedScene,singleSceneURLstring,scraperIDstring,siteIDstring,companystring,URLstring,singeScrapeAdditionalInfostring,limitScrapingbool,masterSiteIdstring,ogSitebool)error {
2625
deferwg.Done()
2726
logScrapeStart(scraperID,siteID)
2827

29-
sceneCollector:=createCollector("badoinkvr.com","babevr.com","vrcosplayx.com","18vr.com")
30-
siteCollector:=createCollector("badoinkvr.com","babevr.com","vrcosplayx.com","18vr.com")
28+
sceneCollector:=createCollector("badoinkvr.com","babevr.com","vrcosplayx.com","18vr.com","realvr.com")
29+
siteCollector:=createCollector("badoinkvr.com","babevr.com","vrcosplayx.com","18vr.com","realvr.com")
30+
3131
trailerCollector:=cloneCollector(sceneCollector)
3232

3333
commonDb,_:=models.GetCommonDB()
@@ -38,16 +38,23 @@ func BadoinkSite(wg *models.ScrapeWG, updateSite bool, knownScenes []string, out
3838
sc:= models.ScrapedScene{}
3939
sc.ScraperID=scraperID
4040
sc.SceneType="VR"
41-
sc.Studio="Badoink"
41+
sc.Studio=company
4242
sc.HomepageURL=strings.Split(e.Request.URL.String(),"?")[0]
43+
sc.MasterSiteId=masterSiteId
4344

4445
// Site ID
4546
sc.Site=siteID
4647

4748
// Scene ID - get from URL
4849
tmp:=strings.Split(sc.HomepageURL,"-")
4950
sc.SiteID=strings.Replace(tmp[len(tmp)-1],"/","",-1)
50-
sc.SceneID=slugify.Slugify(sc.Site)+"-"+sc.SiteID
51+
52+
idPrefix:=scraperID
53+
if!ogSite {
54+
idPrefix="realvr"
55+
}
56+
57+
sc.SceneID=idPrefix+"-"+sc.SiteID
5158

5259
// Title
5360
e.ForEach(`h1.video-title`,func(idint,e*colly.HTMLElement) {
@@ -102,7 +109,7 @@ func BadoinkSite(wg *models.ScrapeWG, updateSite bool, knownScenes []string, out
102109
sc.ActorDetails=make(map[string]models.ActorDetails)
103110
e.ForEach(`a.video-actor-link`,func(idint,e*colly.HTMLElement) {
104111
sc.Cast=append(sc.Cast,strings.TrimSpace(e.Text))
105-
sc.ActorDetails[strings.TrimSpace(e.Text)]= models.ActorDetails{Source:sc.ScraperID+" scrape",ProfileUrl:e.Request.AbsoluteURL(e.Attr("href"))}
112+
sc.ActorDetails[strings.TrimSpace(e.Text)]= models.ActorDetails{Source:idPrefix+" scrape",ProfileUrl:e.Request.AbsoluteURL(e.Attr("href"))}
106113
})
107114

108115
// Date
@@ -268,19 +275,19 @@ func BadoinkSite(wg *models.ScrapeWG, updateSite bool, knownScenes []string, out
268275
}
269276

270277
funcBadoinkVR(wg*models.ScrapeWG,updateSitebool,knownScenes []string,outchan<- models.ScrapedScene,singleSceneURLstring,singeScrapeAdditionalInfostring,limitScrapingbool)error {
271-
returnBadoinkSite(wg,updateSite,knownScenes,out,singleSceneURL,"badoinkvr","BadoinkVR","https://badoinkvr.com/vrpornvideos?order=newest",singeScrapeAdditionalInfo,limitScraping)
278+
returnBadoinkSite(wg,updateSite,knownScenes,out,singleSceneURL,"badoinkvr","BadoinkVR","Badoink","https://badoinkvr.com/vrpornvideos?order=newest",singeScrapeAdditionalInfo,limitScraping,"",true)
272279
}
273280

274281
funcB18VR(wg*models.ScrapeWG,updateSitebool,knownScenes []string,outchan<- models.ScrapedScene,singleSceneURLstring,singeScrapeAdditionalInfostring,limitScrapingbool)error {
275-
returnBadoinkSite(wg,updateSite,knownScenes,out,singleSceneURL,"18vr","18VR","https://18vr.com/vrpornvideos?order=newest",singeScrapeAdditionalInfo,limitScraping)
282+
returnBadoinkSite(wg,updateSite,knownScenes,out,singleSceneURL,"18vr","18VR","Badoink","https://18vr.com/vrpornvideos?order=newest",singeScrapeAdditionalInfo,limitScraping,"",true)
276283
}
277284

278285
funcVRCosplayX(wg*models.ScrapeWG,updateSitebool,knownScenes []string,outchan<- models.ScrapedScene,singleSceneURLstring,singeScrapeAdditionalInfostring,limitScrapingbool)error {
279-
returnBadoinkSite(wg,updateSite,knownScenes,out,singleSceneURL,"vrcosplayx","VRCosplayX","https://vrcosplayx.com/cosplaypornvideos?order=newest",singeScrapeAdditionalInfo,limitScraping)
286+
returnBadoinkSite(wg,updateSite,knownScenes,out,singleSceneURL,"vrcosplayx","VRCosplayX","Badoink","https://vrcosplayx.com/cosplaypornvideos?order=newest",singeScrapeAdditionalInfo,limitScraping,"",true)
280287
}
281288

282289
funcBabeVR(wg*models.ScrapeWG,updateSitebool,knownScenes []string,outchan<- models.ScrapedScene,singleSceneURLstring,singeScrapeAdditionalInfostring,limitScrapingbool)error {
283-
returnBadoinkSite(wg,updateSite,knownScenes,out,singleSceneURL,"babevr","BabeVR","https://babevr.com/vrpornvideos?order=newest",singeScrapeAdditionalInfo,limitScraping)
290+
returnBadoinkSite(wg,updateSite,knownScenes,out,singleSceneURL,"babevr","BabeVR","Badoink","https://babevr.com/vrpornvideos?order=newest",singeScrapeAdditionalInfo,limitScraping,"",true)
284291
}
285292

286293
funcinit() {

‎pkg/scrape/realvr.go

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
package scrape
2+
3+
import (
4+
"strings"
5+
6+
"github.com/xbapps/xbvr/pkg/config"
7+
"github.com/xbapps/xbvr/pkg/models"
8+
)
9+
10+
funcaddRealVRScraper(idstring,namestring,companystring,avatarURLstring,custombool,siteURLstring,masterSiteIdstring) {
11+
suffixedName:=name
12+
siteNameSuffix:=name
13+
ifcustom {
14+
suffixedName+=" (Custom RealVR)"
15+
siteNameSuffix+=" (RealVR)"
16+
}else {
17+
suffixedName+=" (RealVR)"
18+
}
19+
ifavatarURL=="" {
20+
avatarURL="https://realvr.com/icons/realvr/favicon-32x32.png"
21+
}
22+
23+
siteURL=strings.TrimSuffix(siteURL,"/")
24+
siteURL+="/videos/1?order=newest"
25+
26+
ifmasterSiteId=="" {
27+
registerScraper(id,suffixedName,avatarURL,"realvr.com",func(wg*models.ScrapeWG,updateSitebool,knownScenes []string,outchan<- models.ScrapedScene,singleSceneURLstring,singeScrapeAdditionalInfostring,limitScrapingbool)error {
28+
returnBadoinkSite(wg,updateSite,knownScenes,out,singleSceneURL,id,siteNameSuffix,company,siteURL,singeScrapeAdditionalInfo,limitScraping,"",false)
29+
})
30+
}else {
31+
registerAlternateScraper(id,suffixedName,avatarURL,"realvr.com",masterSiteId,func(wg*models.ScrapeWG,updateSitebool,knownScenes []string,outchan<- models.ScrapedScene,singleSceneURLstring,singeScrapeAdditionalInfostring,limitScrapingbool)error {
32+
returnBadoinkSite(wg,updateSite,knownScenes,out,singleSceneURL,id,siteNameSuffix,company,siteURL,singeScrapeAdditionalInfo,limitScraping,masterSiteId,false)
33+
})
34+
}
35+
}
36+
37+
funcinit() {
38+
registerScraper("realvr-single_scene","RealVR - Other Studios","","realvr.com",func(wg*models.ScrapeWG,updateSitebool,knownScenes []string,outchan<- models.ScrapedScene,singleSceneURLstring,singeScrapeAdditionalInfostring,limitScrapingbool)error {
39+
returnBadoinkSite(wg,updateSite,knownScenes,out,singleSceneURL,"","","","",singeScrapeAdditionalInfo,limitScraping,"",false)
40+
})
41+
varscrapers config.ScraperList
42+
scrapers.Load()
43+
for_,scraper:=rangescrapers.XbvrScrapers.RealVRScrapers {
44+
addRealVRScraper(scraper.ID,scraper.Name,scraper.Company,scraper.AvatarUrl,false,scraper.URL,scraper.MasterSiteId)
45+
}
46+
for_,scraper:=rangescrapers.CustomScrapers.RealVRScrapers {
47+
addRealVRScraper(scraper.ID,scraper.Name,scraper.Company,scraper.AvatarUrl,true,scraper.URL,scraper.MasterSiteId)
48+
}
49+
}

‎ui/src/components/RescrapeButton.vue

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,9 @@ export default {
4848
if (this.item.scene_url.toLowerCase().includes("vrporn.com")) {
4949
site="vrporn-single_scene"
5050
}
51+
if (this.item.scene_url.toLowerCase().includes("realvr.com")) {
52+
site="realvr-single_scene"
53+
}
5154
if (this.item.scene_url.toLowerCase().includes("vrphub.com")) {
5255
site="vrphub-single_scene"
5356
}

‎ui/src/views/options/sections/InterfaceAdvanced.vue

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -195,7 +195,7 @@ export default {
195195
this.scraperFieldsValid=false
196196
if (this.scraperName!="") {
197197
if (this.scraperUrl.startsWith("https://")||this.scraperUrl.startsWith("http://") ) {
198-
if (this.scraperUrl.includes("povr.com")||this.scraperUrl.includes("sexlikereal.com")||this.scraperUrl.includes("vrphub.com")||this.scraperUrl.includes("vrporn.com")||this.scraperUrl.includes("stashdb.org")) {
198+
if (this.scraperUrl.includes("povr.com")||this.scraperUrl.includes("sexlikereal.com")||this.scraperUrl.includes("vrphub.com")||this.scraperUrl.includes("vrporn.com")||this.scraperUrl.includes("stashdb.org")||this.scraperUrl.includes("realvr.com")) {
199199
this.scraperFieldsValid=true
200200
}
201201
}

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp