Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commitb3aea59

Browse files
authored
scraper: Fix FuckPassVR native scraper (#1741)
Co-authored-by: crwxaj <crwxaj>
1 parent0746b1a commitb3aea59

File tree

1 file changed

+69
-110
lines changed

1 file changed

+69
-110
lines changed

‎pkg/scrape/fuckpassvr.go

Lines changed: 69 additions & 110 deletions
Original file line numberDiff line numberDiff line change
@@ -3,16 +3,14 @@ package scrape
33
import (
44
"encoding/json"
55
"net/url"
6-
"path"
76
"regexp"
8-
"strconv"
97
"strings"
108
"sync"
119

1210
"github.com/go-resty/resty/v2"
1311
"github.com/gocolly/colly/v2"
12+
"github.com/nleeper/goment"
1413
"github.com/thoas/go-funk"
15-
"github.com/tidwall/gjson"
1614
"github.com/xbapps/xbvr/pkg/models"
1715
)
1816

@@ -23,61 +21,77 @@ func FuckPassVR(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out c
2321
logScrapeStart(scraperID,siteID)
2422

2523
sceneCollector:=createCollector("www.fuckpassvr.com")
24+
siteCollector:=createCollector("www.fuckpassvr.com")
2625

2726
client:=resty.New()
2827
client.SetHeader("User-Agent",UserAgent)
2928

30-
sceneCollector.OnResponse(func(r*colly.Response) {
31-
ifr.StatusCode!=200 {
32-
return
33-
}
34-
res:=gjson.ParseBytes(r.Body)
35-
scenedata:=res.Get("data.scene")
36-
previewVideoURL:=r.Ctx.Get("preview_video_url")
37-
29+
sceneCollector.OnHTML(`html`,func(e*colly.HTMLElement) {
3830
sc:= models.ScrapedScene{}
3931
sc.ScraperID=scraperID
4032
sc.SceneType="VR"
4133
sc.Studio="FuckPassVR"
4234
sc.Site=siteID
35+
sc.HomepageURL=strings.Split(e.Request.URL.String(),"?")[0]
36+
37+
e.ForEach(`meta[property="og:image"]`,func(idint,e*colly.HTMLElement) {
38+
ifid==0 {
39+
url:=strings.Split(e.Request.AbsoluteURL(e.Attr("content")),"?")[0]
40+
re:=regexp.MustCompile(`FPVR(\d+)`)
41+
matches:=re.FindStringSubmatch(url)
42+
iflen(matches)>1 {
43+
sc.SiteID=matches[1]
44+
sc.SceneID="fpvr-"+matches[1]
45+
}
46+
}
47+
})
4348

44-
slug:=scenedata.Get("slug").String()
45-
sc.HomepageURL="https://www.fuckpassvr.com/video/"+slug
46-
47-
sc.SiteID=scenedata.Get("cms_id").String()
48-
ifsc.SiteID==""||!strings.HasPrefix(sc.SiteID,"FPVR") {
49-
return
50-
}
51-
sc.SceneID="fpvr-"+strings.Replace(sc.SiteID,"FPVR","",1)
49+
e.ForEach(`h2.video__title`,func(idint,e*colly.HTMLElement) {
50+
ifid==0 {
51+
sc.Title=strings.TrimSpace(e.Text)
52+
}
53+
})
5254

53-
sc.Released=scenedata.Get("active_schedule").String()[:10]
54-
sc.Title=scenedata.Get("name").String()
55-
sc.Duration=int(scenedata.Get("duration").Int())
56-
sc.Covers=append(sc.Covers,scenedata.Get("thumbnail_url").String())
55+
e.ForEach(`web-vr-video-player`,func(idint,e*colly.HTMLElement) {
56+
sc.Covers=append(sc.Covers,strings.Trim(e.Attr("coverimage")," '"))
57+
})
5758

58-
desc:=scenedata.Get("description").String()
59-
desc=strings.ReplaceAll(desc,"<p>","")
60-
desc=strings.ReplaceAll(desc,"</p>","\n\n")
61-
re:=regexp.MustCompile(`<(.|\n)*?>`)// strip_tags
62-
sc.Synopsis=re.ReplaceAllString(desc,"")
59+
e.ForEach(`div.profile__gallery a.profile__galleryElement`,func(idint,e*colly.HTMLElement) {
60+
sc.Gallery=append(sc.Gallery,strings.TrimSpace(e.Attr("href")))
61+
})
6362

6463
sc.ActorDetails=make(map[string]models.ActorDetails)
65-
scenedata.Get("porn_star_lead").ForEach(func(_,star gjson.Result)bool {
66-
name:=star.Get("name").String()
67-
sc.Cast=append(sc.Cast,name)
68-
sc.ActorDetails[name]= models.ActorDetails{Source:sc.ScraperID+" scrape",ProfileUrl:"https://www.fuckpassvr.com/api/api/seo?porn_star_slug="+star.Get("slug").String()}
69-
returntrue
64+
e.ForEach(`div.models a`,func(idint,e*colly.HTMLElement) {
65+
sc.Cast=append(sc.Cast,strings.TrimSpace(e.Attr("title")))
66+
sc.ActorDetails[strings.TrimSpace(e.Attr("title"))]= models.ActorDetails{Source:sc.ScraperID+" scrape",ProfileUrl:e.Request.AbsoluteURL(e.Attr("href"))}
7067
})
71-
scenedata.Get("porn_star").ForEach(func(_,star gjson.Result)bool {
72-
name:=star.Get("name").String()
73-
sc.Cast=append(sc.Cast,name)
74-
sc.ActorDetails[name]= models.ActorDetails{Source:sc.ScraperID+" scrape",ProfileUrl:"https://www.fuckpassvr.com/api/api/seo?porn_star_slug="+star.Get("slug").String()}
75-
returntrue
68+
69+
e.ForEach(`a.tag`,func(idint,e*colly.HTMLElement) {
70+
sc.Tags=append(sc.Tags,strings.TrimSpace(e.Attr("title")))
7671
})
7772

78-
scenedata.Get("tag_input").ForEach(func(_,tag gjson.Result)bool {
79-
sc.Tags=append(sc.Tags,tag.String())
80-
returntrue
73+
e.ForEach(`div.readMoreWrapper2`,func(idint,e*colly.HTMLElement) {
74+
sc.Synopsis=strings.TrimSpace(e.Text)
75+
})
76+
77+
e.ForEach(`div.video__addons p.wrapper__text`,func(idint,e*colly.HTMLElement) {
78+
s:=strings.TrimSpace(e.Text)
79+
ifstrings.HasPrefix(s,"Released:") {
80+
tmpDate,_:=goment.New(strings.TrimSpace(strings.TrimPrefix(s,"Released:")),"MMM DD YYYY")
81+
sc.Released=tmpDate.Format("YYYY-MM-DD")
82+
}
83+
})
84+
85+
e.ForEach(`div.wrapper__download a.wrapper__downloadLink`,func(idint,e*colly.HTMLElement) {
86+
url,err:=url.Parse(e.Attr("href"))
87+
iferr==nil {
88+
parts:=strings.Split(url.Path,"/")
89+
iflen(parts)>0 {
90+
fn:=parts[len(parts)-1]
91+
fn=strings.Replace(fn,"2min","FULL",-1)
92+
sc.Filenames=append(sc.Filenames,fn)
93+
}
94+
}
8195
})
8296

8397
// trailer details
@@ -86,82 +100,27 @@ func FuckPassVR(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out c
86100
strParams,_:=json.Marshal(params)
87101
sc.TrailerSrc=string(strParams)
88102

89-
resolutions:= []string{"8kUHD","8kHD","4k","2k","1k"}
90-
parsedFileNameURL,err:=url.Parse(previewVideoURL)
91-
iferr==nil {
92-
fileNameBase:=path.Base(parsedFileNameURL.Path)
93-
ifstrings.HasSuffix(strings.ToLower(fileNameBase),"_rollover.mp4") {
94-
fori:=rangeresolutions {
95-
fn:=fileNameBase[:len(fileNameBase)-len("_rollover.mp4")]+"-FULL_"+resolutions[i]+".mp4"
96-
sc.Filenames=append(sc.Filenames,fn)
97-
}
98-
}
99-
}else {
100-
log.Error(err)
101-
}
103+
out<-sc
104+
})
102105

103-
resp,err:=client.R().
104-
SetQueryParams(map[string]string{
105-
"scene_id":scenedata.Get("id").String(),
106-
}).
107-
Get("https://www.fuckpassvr.com/api/api/storyboard/show")
108-
109-
iferr==nil {
110-
res:=gjson.ParseBytes(resp.Body())
111-
res.Get("data.storyboards.#.image_origin_url").ForEach(func(_,url gjson.Result)bool {
112-
sc.Gallery=append(sc.Gallery,url.String())
113-
returntrue
114-
})
115-
}else {
116-
log.Error(err)
106+
siteCollector.OnHTML(`section.pagination a`,func(e*colly.HTMLElement) {
107+
if!limitScraping {
108+
siteCollector.Visit(e.Attr("href"))
117109
}
118-
119-
out<-sc
120110
})
121111

122-
varpageint64=1
123-
varlastPageint64=1
124-
iflimitScraping {
125-
lastPage=1
126-
}
112+
siteCollector.OnHTML(`div.videos__element a.videos__videoTitle`,func(e*colly.HTMLElement) {
113+
sceneURL:=e.Request.AbsoluteURL(e.Attr("href"))
114+
115+
if!funk.ContainsString(knownScenes,sceneURL) {
116+
sceneCollector.Visit(sceneURL)
117+
}
118+
})
127119

128120
ifsingleSceneURL!="" {
129-
ctx:=colly.NewContext()
130-
ctx.Put("preview_video_url","")
131-
slug:=strings.Replace(singleSceneURL,"https://www.fuckpassvr.com/video/","",1)
132-
sceneDetail:="https://www.fuckpassvr.com/api/api/scene/show?slug="+slug
133-
sceneCollector.Request("GET",sceneDetail,nil,ctx,nil)
121+
sceneCollector.Visit(singleSceneURL)
134122
}else {
135-
forpage<=lastPage {
136-
resp,err:=client.R().
137-
SetQueryParams(map[string]string{
138-
"size":"24",
139-
"sortBy":"newest",
140-
"page":strconv.FormatInt(page,10),
141-
}).
142-
Get("https://www.fuckpassvr.com/api/api/scene")
143-
144-
iferr==nil {
145-
res:=gjson.ParseBytes(resp.Body())
146-
res.Get("data.scenes.data").ForEach(func(_,scenedata gjson.Result)bool {
147-
ctx:=colly.NewContext()
148-
ctx.Put("preview_video_url",scenedata.Get("preview_video_url").String())
149-
150-
sceneURL:="https://www.fuckpassvr.com/video/"+scenedata.Get("slug").String()
151-
sceneDetail:="https://www.fuckpassvr.com/api/api/scene/show?slug="+scenedata.Get("slug").String()
152-
153-
if!funk.ContainsString(knownScenes,sceneURL) {
154-
sceneCollector.Request("GET",sceneDetail,nil,ctx,nil)
155-
}
156-
157-
returntrue
158-
})
159-
lastPage=res.Get("data.scenes.last_page").Int()
160-
page=page+1
161-
}else {
162-
log.Error(err)
163-
}
164-
}
123+
siteCollector.Visit("https://www.fuckpassvr.com/destination")
165124
}
166125

167126
ifupdateSite {

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp