Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commite98f7fe

Browse files
authored
scraper: Fix KinkVR scraper (#1875)
* Final KinkVR workingFile names are dropped upon migration. Appears to have to good means for the future of guessing the filename. Do not need migration code. Due to the change in the URL all scenes are resraped on next update and all scraped data is replaced with its new format* Bug fixBootstrap sites use a trailing slash. I remove this for internal logic and this is what is saved. But when comparing new scenes to what has been search the trailing slash caused this logic to never go true.* Spin off kinkvr completelyAfter some digging it appears kinkvr was sold and not a website redesign by badoink. Now completly separates kinkvr from badoink* Favicon Update* Minor Code Clean upMissed some remnants of badoinks colab with kink. Also removed scraper for actor has no info for actor is on kink just links to other scenes
1 parent1a33282 commite98f7fe

File tree

3 files changed

+128
-9
lines changed

3 files changed

+128
-9
lines changed

‎pkg/models/model_external_reference.go

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -710,8 +710,6 @@ func (scrapeRules ActorScraperConfig) buildGenericActorScraperRules() {
710710
scrapeRules.GenericActorScrapingConfig["vrcosplayx scrape"]=siteDetails
711711
siteDetails.Domain="18vr.com"
712712
scrapeRules.GenericActorScrapingConfig["18vr scrape"]=siteDetails
713-
siteDetails.Domain="kinkvr.com"
714-
scrapeRules.GenericActorScrapingConfig["kinkvr scrape"]=siteDetails
715713

716714
siteDetails=GenericScraperRuleSet{}
717715
siteDetails.Domain="darkroomvr.com"

‎pkg/scrape/badoink.go

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -26,8 +26,8 @@ func BadoinkSite(wg *models.ScrapeWG, updateSite bool, knownScenes []string, out
2626
deferwg.Done()
2727
logScrapeStart(scraperID,siteID)
2828

29-
sceneCollector:=createCollector("badoinkvr.com","babevr.com","vrcosplayx.com","18vr.com","kinkvr.com")
30-
siteCollector:=createCollector("badoinkvr.com","babevr.com","vrcosplayx.com","18vr.com","kinkvr.com")
29+
sceneCollector:=createCollector("badoinkvr.com","babevr.com","vrcosplayx.com","18vr.com")
30+
siteCollector:=createCollector("badoinkvr.com","babevr.com","vrcosplayx.com","18vr.com")
3131
trailerCollector:=cloneCollector(sceneCollector)
3232

3333
commonDb,_:=models.GetCommonDB()
@@ -283,14 +283,9 @@ func BabeVR(wg *models.ScrapeWG, updateSite bool, knownScenes []string, out chan
283283
returnBadoinkSite(wg,updateSite,knownScenes,out,singleSceneURL,"babevr","BabeVR","https://babevr.com/vrpornvideos?order=newest",singeScrapeAdditionalInfo,limitScraping)
284284
}
285285

286-
funcKinkVR(wg*models.ScrapeWG,updateSitebool,knownScenes []string,outchan<- models.ScrapedScene,singleSceneURLstring,singeScrapeAdditionalInfostring,limitScrapingbool)error {
287-
returnBadoinkSite(wg,updateSite,knownScenes,out,singleSceneURL,"kinkvr","KinkVR","https://kinkvr.com/bdsm-vr-videos?order=newest",singeScrapeAdditionalInfo,limitScraping)
288-
}
289-
290286
funcinit() {
291287
registerScraper("badoinkvr","BadoinkVR","https://pbs.twimg.com/profile_images/618071358933610497/QaMV81nF_200x200.png","badoinkvr.com",BadoinkVR)
292288
registerScraper("18vr","18VR","https://pbs.twimg.com/profile_images/989481761783545856/w-iKqgqV_200x200.jpg","18vr.com",B18VR)
293289
registerScraper("vrcosplayx","VRCosplayX","https://pbs.twimg.com/profile_images/900675974039298049/ofMytpkQ_200x200.jpg","vrcosplayx.com",VRCosplayX)
294290
registerScraper("babevr","BabeVR","https://babevr.com/icons/babevr/apple-touch-icon.png","babevr.com",BabeVR)
295-
registerScraper("kinkvr","KinkVR","https://kinkvr.com/icons/kinkvr/apple-touch-icon.png","kinkvr.com",KinkVR)
296291
}

‎pkg/scrape/kinkvr.go

Lines changed: 126 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,126 @@
1+
package scrape
2+
3+
import (
4+
"strings"
5+
6+
"github.com/gocolly/colly/v2"
7+
"github.com/mozillazg/go-slugify"
8+
"github.com/nleeper/goment"
9+
"github.com/thoas/go-funk"
10+
"github.com/xbapps/xbvr/pkg/models"
11+
)
12+
13+
funcKinkVR(wg*models.ScrapeWG,updateSitebool,knownScenes []string,outchan<- models.ScrapedScene,singleSceneURLstring,singeScrapeAdditionalInfostring,limitScrapingbool)error {
14+
deferwg.Done()
15+
scraperID:="kinkvr"
16+
siteID:="KinkVR"
17+
logScrapeStart(scraperID,siteID)
18+
19+
sceneCollector:=createCollector("kinkvr.com")
20+
siteCollector:=createCollector("kinkvr.com")
21+
22+
// These cookies are needed for age verification.
23+
siteCollector.OnRequest(func(r*colly.Request) {
24+
r.Headers.Set("Cookie","agreedToDisclaimer=true")
25+
})
26+
27+
sceneCollector.OnRequest(func(r*colly.Request) {
28+
r.Headers.Set("Cookie","agreedToDisclaimer=true")
29+
})
30+
31+
sceneCollector.OnHTML(`html`,func(e*colly.HTMLElement) {
32+
sc:= models.ScrapedScene{}
33+
sc.ScraperID=scraperID
34+
sc.SceneType="VR"
35+
sc.Studio="Badoink"
36+
sc.Site=siteID
37+
sc.SiteID=""
38+
sc.HomepageURL=e.Request.URL.String()
39+
40+
// Cover Url
41+
coverURL:=e.ChildAttr("div#povVideoContainer dl8-video","poster")
42+
sc.Covers=append(sc.Covers,coverURL)
43+
44+
// Gallery
45+
e.ForEach(`div.owl-carousel div.item`,func(idint,e*colly.HTMLElement) {
46+
sc.Gallery=append(sc.Gallery,e.ChildAttr("img","src"))
47+
})
48+
49+
// Incase we scrape a single scene use one of the gallery images for the cover
50+
ifsingleSceneURL!="" {
51+
sc.Covers=append(sc.Covers,sc.Gallery[0])
52+
}
53+
54+
// Cast
55+
sc.ActorDetails=make(map[string]models.ActorDetails)
56+
e.ForEach(`table.video-description-list tbody`,func(idint,e*colly.HTMLElement) {
57+
// Cast
58+
e.ForEach(`tr:nth-child(1) a`,func(idint,e*colly.HTMLElement) {
59+
ifstrings.TrimSpace(e.Text)!="" {
60+
sc.Cast=append(sc.Cast,strings.TrimSpace(e.Text))
61+
sc.ActorDetails[strings.TrimSpace(e.Text)]= models.ActorDetails{ProfileUrl:e.Request.AbsoluteURL(e.Attr("href"))}
62+
}
63+
})
64+
65+
// Tags
66+
e.ForEach(`tr:nth-child(2) a`,func(idint,e*colly.HTMLElement) {
67+
tag:=strings.TrimSpace(e.Text)
68+
sc.Tags=append(sc.Tags,tag)
69+
})
70+
71+
// Date
72+
tmpDate,_:=goment.New(strings.TrimSpace(e.ChildText(`tr:nth-child(3) td:last-child`)),"MMMM DD, YYYY")
73+
sc.Released=tmpDate.Format("YYYY-MM-DD")
74+
})
75+
76+
// Synposis
77+
sc.Synopsis=strings.TrimSpace(e.ChildText("div.accordion-body"))
78+
79+
// Title
80+
sc.Title=e.ChildText("h1.page-title")
81+
82+
// Scene ID -- Uses the ending number of the video url instead of the ID used for the directory that the video link is stored in(Maintains backwards compatibility with old scenes)
83+
tmpUrlStr,_:=strings.CutSuffix(e.Request.URL.String(),"/")
84+
tmp:=strings.Split(tmpUrlStr,"/")
85+
siteIDstr:=strings.Split(tmp[len(tmp)-1],"-")
86+
sc.SiteID=siteIDstr[len(siteIDstr)-1]
87+
88+
ifsc.SiteID!="" {
89+
sc.SceneID=slugify.Slugify(sc.Site)+"-"+sc.SiteID
90+
91+
// save only if we got a SceneID
92+
out<-sc
93+
}
94+
})
95+
96+
siteCollector.OnHTML(`a.page-link[aria-label="Next"]:not(.disabled)`,func(e*colly.HTMLElement) {
97+
if!limitScraping {
98+
pageURL:=e.Request.AbsoluteURL(e.Attr("href"))
99+
siteCollector.Visit(pageURL)
100+
}
101+
})
102+
103+
siteCollector.OnHTML(`div.video-grid-view a`,func(e*colly.HTMLElement) {
104+
sceneURL:=e.Request.AbsoluteURL(e.Attr("href"))
105+
// If scene exist in database, there's no need to scrape
106+
if!funk.ContainsString(knownScenes,sceneURL) {
107+
sceneCollector.Visit(sceneURL)
108+
}
109+
})
110+
111+
ifsingleSceneURL!="" {
112+
sceneCollector.Visit(singleSceneURL)
113+
}else {
114+
siteCollector.Visit("https://kinkvr.com/videos/page1")
115+
}
116+
117+
ifupdateSite {
118+
updateSiteLastUpdate(scraperID)
119+
}
120+
logScrapeFinished(scraperID,siteID)
121+
returnnil
122+
}
123+
124+
funcinit() {
125+
registerScraper("kinkvr","KinkVR","https://static.rlcontent.com/shared/KINK/skins/web-10/branding/favicon.png","kinkvr.com",KinkVR)
126+
}

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp