async def async_get_info_from_url(url) -> str:
user_agent = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; rv:91.0) Gecko/20100101 Firefox/91.0'}
async with aiohttp.ClientSession() as session:
async with session.get(url, headers=user_agent) as response:
content = await response.text()
tree = html.fromstring(content)
info = tree.xpath(xpath)
return info[0] if info else None
async def main():
urls = read_urls_from_file('boj.txt')
tasks = [async_get_info_from_url(url) for url in urls]
results = await asyncio.gather(*tasks)
return results
if __name__ == '__main__':
results = asyncio.run(main())
url_info = {}
for url, result in zip(read_urls_from_file('boj.txt'), results):
url_info[url] = int(result) if result is not None else 0
sorted_urls = sorted(url_info.items(), key=lambda item: item[1], reverse=True)
for url, info in sorted_urls:
print(f'{info} {url}')
sorted_urls = [url for url, _ in sorted_urls]
write_urls_to_file('sorted-boj.txt', sorted_urls)
백준 문제 많이 제출한순으로 풀려고 스크립트 짬.
원래는 비동기 없이 짰는데
몇백문제 넘어가니까 너무 느려서 바꿨음 ㅇㅅㅇ,,
자스랑 완전 비슷하네요,,,
네. 이상입니다.
좋은 하루되세요.
이거 스크립트 한방에 리퀘스트 수만개 날아가는거 아님? ㅇㅅㅇ asyncio.Semaphore로 max concurrency 설정하셈
오 땡큐.. 한번 해볼게 ~.~
https://www.scrapehero.com/how-to-prevent-getting-blacklisted-while-scraping/
이거 읽어봐
https://webscraping.com/blog/How-to-crawl-websites-without-being-blocked/
이것두
ㅇㅅㅇ..
ㅇㅅㅇ..
고생이 많다