Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commite18d854

Browse files
authored
Merge pull request#31 from juzim/error-notifications
Error notifications
2 parents24d4725 +0cb84d8 commite18d854

File tree

8 files changed

+136
-53
lines changed

8 files changed

+136
-53
lines changed

‎README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ This crawler automates the following step:
1111
* download source code and book cover
1212
* upload files to Google Drive or via scp
1313
* store data on Firebase
14-
* notify via email, IFTTT or Join
14+
* notify via email, IFTTT or Join (on success and errors)
1515
* schedule daily job on Heroku or with Docker
1616

1717
###Default command

‎script/notification/gmail.py‎

Lines changed: 48 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,41 @@ def __prepare_message(self):
5555

5656
returnmsg
5757

58+
def__prepare_error_message(self,exception,source):
59+
"""
60+
"""
61+
#log_json(self.__packpub_info)
62+
#log_json(self.__upload_info)
63+
64+
msg=MIMEMultipart('alternative')
65+
msg['Subject']="[packtpub-crawler]"
66+
msg['From']=self.__config.get('gmail','gmail.from')
67+
msg['To']=self.__config.get('gmail','gmail.to')
68+
69+
text="Error downloading today's ebook [{source}]".format(source=source)
70+
html="""\
71+
<html>
72+
<head></head>
73+
<body>
74+
<div>{title}</div>
75+
<div>{description}</div>
76+
""".format(title=text,
77+
description=repr(exception))
78+
79+
html+="""\
80+
<div>Powered by <a href="https://github.com/niqdev/packtpub-crawler">packtpub-crawler</a></div>
81+
</body>
82+
</html>
83+
"""
84+
85+
part1=MIMEText(text,'plain')
86+
part2=MIMEText(html,'html')
87+
88+
msg.attach(part1)
89+
msg.attach(part2)
90+
91+
returnmsg
92+
5893
defsend(self):
5994
server=smtplib.SMTP(self.__config.get('gmail','gmail.host'),self.__config.get('gmail','gmail.port'))
6095
server.starttls()
@@ -65,4 +100,16 @@ def send(self):
65100
server.sendmail(message['From'],receivers,message.as_string())
66101
server.quit()
67102

68-
log_success('[+] Notified to: {0}'.format(receivers))
103+
log_success('[+] notified to: {0}'.format(receivers))
104+
105+
defsendError(self,exception,source):
106+
server=smtplib.SMTP(self.__config.get('gmail','gmail.host'),self.__config.get('gmail','gmail.port'))
107+
server.starttls()
108+
server.login(self.__config.get('gmail','gmail.username'),self.__config.get('gmail','gmail.password'))
109+
110+
message=self.__prepare_error_message(exception,source)
111+
receivers=message['To'].split(",")
112+
server.sendmail(message['From'],receivers,message.as_string())
113+
server.quit()
114+
115+
log_success('[+] error notifikation sent to: {0}'.format(receivers))

‎script/notification/ifttt.py‎

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -6,15 +6,18 @@ class Ifttt(object):
66
"""
77

88
def__init__(self,config,packpub_info,upload_info):
9-
self.__config=config
109
self.__packpub_info=packpub_info
10+
self.__url="https://maker.ifttt.com/trigger/{eventName}/with/key/{apiKey}".format(
11+
eventName=config.get('ifttt','ifttt.event_name'),
12+
apiKey=config.get('ifttt','ifttt.key')
13+
)
1114

1215
defsend(self):
13-
url="https://maker.ifttt.com/trigger/{eventName}/with/key/{apiKey}".format(
14-
eventName=self.__config.get('ifttt','ifttt.event_name'),
15-
apiKey=self.__config.get('ifttt','ifttt.key')
16-
)
16+
r=requests.post(self.__url,data= {'value1':self.__packpub_info['title'].encode('utf-8'),'value2':self.__packpub_info['description'].encode('utf-8')})
17+
log_success('[+] notification sent to IFTTT')
1718

18-
r=requests.post(url,data= {'value1':self.__packpub_info['title'].encode('utf-8'),'value2':self.__packpub_info['description'].encode('utf-8')})
19+
defsendError(self,exception,source):
20+
title="packtpub-crawler [{source}]: Could not download ebook".format(source=source)
21+
r=requests.post(self.__url,data= {'value1':title,'value2':repr(exception)})
1922

20-
log_success('[+]Notification sent to IFTTT')
23+
log_success('[+]error notification sent to IFTTT')

‎script/notification/join.py‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,4 +19,16 @@ def send(self):
1919

2020
r=requests.post(url)
2121

22-
log_success('[+] Notification sent to Join')
22+
log_success('[+] notification sent to Join')
23+
24+
defsendError(self,exception,source):
25+
url="https://joinjoaomgcd.appspot.com/_ah/api/messaging/v1/sendPush?apikey={apiKey}&deviceId={deviceIds}&title={title}&text={description}".format(
26+
apiKey=self.__config.get('join','join.api_key'),
27+
deviceIds=self.__config.get('join','join.device_ids'),
28+
title='packtpub-crawler {source}: Could not download ebook'.format(source=source),
29+
description=repr(exception)
30+
)
31+
32+
r=requests.post(url)
33+
34+
log_success('[+] error notification sent to Join')

‎script/notify.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,3 +28,9 @@ def run(self):
2828
"""
2929
"""
3030
self.service.send()
31+
32+
33+
defsendError(self,exception,source):
34+
"""
35+
"""
36+
self.service.sendError(exception,source)

‎script/packtpub.py‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,8 @@ def __init_headers(self):
2929
}
3030

3131
def__log_response(self,response,method='GET',detail=False):
32-
print'[-] {0} {1} | {2}'.format(method,response.url,response.status_code)
3332
ifdetail:
33+
print'[-] {0} {1} | {2}'.format(method,response.url,response.status_code)
3434
print'[-] cookies:'
3535
log_dict(requests.utils.dict_from_cookiejar(self.__session.cookies))
3636
print'[-] headers:'
@@ -94,6 +94,9 @@ def __GET_claim(self):
9494
soup=make_soup(response)
9595
div_target=soup.find('div', {'id':'product-account-list'})
9696

97+
ifdiv_targetisNone:
98+
raiseException('Could not access claim page. This is most likely caused by invalid credentials')
99+
97100
# only last one just claimed
98101
div_claimed_book=div_target.select('.product-line')[0]
99102
self.info['book_id']=div_claimed_book['nid']
@@ -108,11 +111,11 @@ def run(self):
108111
"""
109112

110113
self.__GET_login()
111-
wait(self.__delay)
114+
wait(self.__delay,self.__dev)
112115
self.__POST_login()
113-
wait(self.__delay)
116+
wait(self.__delay,self.__dev)
114117
self.__GET_claim()
115-
wait(self.__delay)
118+
wait(self.__delay,self.__dev)
116119

117120
defdownload_ebooks(self,types):
118121
"""

‎script/spider.py‎

Lines changed: 48 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,44 @@ def parse_types(args):
1515
else:
1616
returnargs.types
1717

18+
defrun(packpub,args,config):
19+
packpub.run()
20+
21+
ifargs.dev:
22+
log_json(packpub.info)
23+
24+
log_success('[+] book successfully claimed')
25+
26+
upload=None
27+
upload_info=None
28+
packpub_info=None
29+
30+
ifnotargs.claimOnly:
31+
types=parse_types(args)
32+
33+
packpub.download_ebooks(types)
34+
35+
ifargs.extras:
36+
packpub.download_extras()
37+
38+
ifargs.archive:
39+
raiseNotImplementedError('not implemented yet!')
40+
41+
ifargs.uploadisnotNone:
42+
upload=Upload(config,args.upload)
43+
upload.run(packpub.info['paths'])
44+
45+
ifuploadisnotNoneanduploadisnotSERVICE_DRIVE:
46+
log_warn('[-] skip store info: missing upload info')
47+
elifargs.storeisnotNone:
48+
Database(config,args.store,packpub.info,upload.info).store()
49+
50+
ifargs.notify:
51+
ifuploadisnotNone:
52+
upload_info=upload.info
53+
54+
Notify(config,packpub.info,upload_info,args.notify).run()
55+
1856
defmain():
1957
parser=argparse.ArgumentParser(
2058
description='Download FREE eBook every day from www.packtpub.com',
@@ -41,52 +79,25 @@ def main():
4179
now=datetime.datetime.now()
4280
log_info('[*] {date} - Fetching today\'s books'.format(date=now.strftime("%Y-%m-%d %H:%M")))
4381

82+
packtpub=None
83+
4484
try:
45-
#ip_address()
4685
config=config_file(args.config)
47-
types=parse_types(args)
4886

87+
#ip_address()
88+
log_info('[*] getting daily free ebook')
4989
packpub=Packpub(config,args.dev)
50-
packpub.run()
51-
52-
ifargs.dev:
53-
log_json(packpub.info)
54-
55-
log_success('[+] book successfully claimed')
56-
57-
upload=None
58-
59-
ifnotargs.claimOnly:
60-
packpub.download_ebooks(types)
61-
62-
ifargs.extras:
63-
packpub.download_extras()
64-
65-
ifargs.archive:
66-
raiseNotImplementedError('not implemented yet!')
67-
68-
ifargs.uploadisnotNone:
69-
upload=Upload(config,args.upload)
70-
upload.run(packpub.info['paths'])
71-
72-
ifuploadisnotNoneanduploadisnotSERVICE_DRIVE:
73-
log_warn('[-] skip store info: missing upload info')
74-
elifargs.storeisnotNone:
75-
Database(config,args.store,packpub.info,upload.info).store()
76-
77-
ifargs.notify:
78-
upload_info=None
79-
80-
ifuploadisnotNone:
81-
upload_info=upload.info
82-
83-
Notify(config,packpub.info,upload_info,args.notify).run()
90+
run(packpub,args,config)
8491

8592
exceptKeyboardInterrupt:
8693
log_error('[-] interrupted manually')
94+
8795
exceptExceptionase:
8896
log_debug(e)
89-
log_error('[-] something weird occurred, exiting...')
97+
ifargs.notify:
98+
Notify(config,None,None,args.notify).sendError(e,'global')
99+
100+
log_info('[*] done')
90101

91102
if__name__=='__main__':
92103
print ("""

‎script/utils.py‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -39,9 +39,10 @@ def make_soup(response, debug=False):
3939
printsoup.prettify().encode('utf-8')
4040
returnsoup
4141

42-
defwait(delay):
42+
defwait(delay,isDev):
4343
ifdelay>0:
44-
print'[-] going to sleep {0} seconds'.format(delay)
44+
ifisDev:
45+
print'[-] going to sleep {0} seconds'.format(delay)
4546
sleep(delay)
4647

4748
defdownload_file(r,url,directory,filename,headers):

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp