Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commit1a817bc

Browse files
committed
Modification in web scraping scenarios
1 parent3de7c15 commit1a817bc

File tree

2 files changed

+12
-6
lines changed

2 files changed

+12
-6
lines changed

‎tests/web-scraping/test_async_ecommerce_scrapping.py

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,11 @@
33
importsys
44
importssl
55
importcertifi
6+
importtime
67
frompprintimportpprint
78
frombs4importBeautifulSoup
89

9-
sys.path.append(sys.path[0]+"/../../..")
10+
sys.path.append(sys.path[0]+"/../../")
1011

1112
frompageobject.locatorsimportlocators
1213
frompageobject.helpersimporthelpers
@@ -25,7 +26,7 @@ async def fetch(url, session):
2526

2627
asyncdefscrap_ecommerce(url):
2728
ssl_context=ssl.create_default_context(cafile=certifi.where())
28-
asyncwithaiohttp.ClientSession(connector=aiohttp.TCPConnector(ssl_context=ssl_context))assession:
29+
asyncwithaiohttp.ClientSession(connector=aiohttp.TCPConnector(ssl=ssl_context))assession:
2930
html=awaitfetch(url,session)
3031
soup=BeautifulSoup(html,'html.parser')
3132

@@ -48,6 +49,7 @@ async def scrap_ecommerce(url):
4849
returnmeta_data_arr
4950

5051
asyncdefmain():
52+
start_time=time.time()
5153
base_url=locators.test_bs4_url
5254
tasks= [scrap_ecommerce(f"{base_url}&page={i}")foriinrange(1,6)]
5355
results=awaitasyncio.gather(*tasks)
@@ -57,7 +59,8 @@ async def main():
5759
print("*********************************************************************************************************")
5860
helpers.print_scrapped_content(result)
5961
print()
62+
63+
print("\nTime elapsed is "+str((time.time()-start_time))+" seconds")
6064

6165
if__name__=='__main__':
62-
output=asyncio.run(main())
63-
print(output)
66+
output=asyncio.run(main())

‎tests/web-scraping/test_sync_ecommerce_scrapping.py

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
# Import the locators file
44
importsys
55
frompprintimportpprint
6-
sys.path.append(sys.path[0]+"/../../..")
6+
sys.path.append(sys.path[0]+"/../../")
77

88
frompageobject.locatorsimportlocators
99
frompageobject.locatorsimport*
@@ -44,10 +44,13 @@ def scrap_ecommerce(url) -> list:
4444
# Page 1: https://ecommerce-playground.lambdatest.io/index.php?route=product/category&path=57&page=1
4545
# Page 5: https://ecommerce-playground.lambdatest.io/index.php?route=product/category&path=57&page=5
4646
if__name__=='__main__':
47+
start_time=time.time()
4748
foriterationinrange(1,6):
4849
test_url=locators.test_bs4_url+"&page="+str(iteration)
4950
meta_data_arr=scrap_ecommerce(test_url)
5051
print('\n')
5152
print("Product Page = "+test_url)
5253
print("*********************************************************************************************************\n")
53-
helpers.print_scrapped_content(meta_data_arr)
54+
helpers.print_scrapped_content(meta_data_arr)
55+
56+
print("\nTime elapsed is "+str((time.time()-start_time))+" seconds")

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp