发表于 | 分类于

之前有整理过Selenium的使用，这里重新用python整理一下

对象选择

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
from selenium import webdriver
from selenium.webdriver.common.by import By

browser = webdriver.Chrome()
browser.get('https://www.taobao.com')
'''
 		ID = "id"
    XPATH = "xpath"
    LINK_TEXT = "link text"
    PARTIAL_LINK_TEXT = "partial link text"
    NAME = "name"
    TAG_NAME = "tag name"
    CLASS_NAME = "class name"
    CSS_SELECTOR = "css selector"
'''
input_first = browser.find_element(By.ID, 'q')
browser.close()

节点交互

1
2
3
4
# 节点交互
input_first.click()
input_first.send_keys('111') # 输入字符
input_first.clear() # 清空内容
阅读全文 »

发表于 | 分类于 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
import asyncio
import logging
import aiohttp
from motor.motor_asyncio import AsyncIOMotorClient

logging.basicConfig(level=logging.INFO,
                    format='%(asctime)s - %(levelname)s: %(message)s')
# mongodb 配置
MONGO_CONNECTION_STRING = 'mongodb://root:yilvs2021@192.168.0.158:27017'
client = AsyncIOMotorClient(MONGO_CONNECTION_STRING)
db = client.test
collection = db.asyncmovies

INDEX_URL = 'https://spa5.scrape.center/api/book/?limit=18&offset={offset}'
DETAIL_URL = 'https://spa5.scrape.center/api/book/{id}'

semaphore = asyncio.Semaphore(10)
session = None  # type: aiohttp.client.ClientSession

async def get_all_page(page_num):
    offset = 18 * page_num
    url_format = INDEX_URL.format(offset=offset)
    logging.info('url %s', url_format)
    return await url_request(url_format)


async def get_all_page_details(_id):
    url_format = DETAIL_URL.format(id=_id)
    logging.info('url %s', url_format)
    response = await url_request(url_format)
    try:
        logging.info('saving data %s', response.get('id'))
        await collection.insert_one(response)
    except AttributeError:
        logging.error('', exc_info=True)


async def url_request(url):
    async with semaphore:
        try:
            async with session.get(url) as response:
                return await response.json()
        except aiohttp.ClientError:
            logging.error('error occurred while scraping %s', url, exc_info=True)


async def main():
    global session
    session = aiohttp.ClientSession()
    # 创建100个协程去调用request方法
    tasks = [asyncio.ensure_future(get_all_page(page_num)) for page_num in range(50)]
    gather_tasks = await asyncio.gather(*tasks)
    ids = []
    for task in gather_tasks:  # type: dict
        task_results = task.get('results')
        for item in task_results:
            ids.append(item.get('id'))
    # 调用详情页面
    detail_tasks = [asyncio.ensure_future(get_all_page_details(_id)) for _id in ids]
    await asyncio.gather(*detail_tasks)
    await session.close()


if __name__ == '__main__':
    asyncio.get_event_loop().run_until_complete(main())

请求偶尔会出现的问题，900条数据报错4条

1
2
3
4
5
6
7
8
9
10
11
12
13
2022-03-07 14:45:30,663 - ERROR: error occurred while scraping https://spa5.scrape.center/api/book/4903440
Traceback (most recent call last):
  File "/Users/keeep/Downloads/py-project/pythonSpider/ScrapeSpa5/myspider.py", line 42, in url_request
    async with session.get(url) as response:
  File "/Users/keeep/Downloads/py-project/pythonSpider/venv/lib/python3.8/site-packages/aiohttp/client.py", line 1138, in __aenter__
    self._resp = await self._coro
  File "/Users/keeep/Downloads/py-project/pythonSpider/venv/lib/python3.8/site-packages/aiohttp/client.py", line 559, in _request
    await resp.start(conn)
  File "/Users/keeep/Downloads/py-project/pythonSpider/venv/lib/python3.8/site-packages/aiohttp/client_reqrep.py", line 898, in start
    message, payload = await protocol.read()  # type: ignore[union-attr]
  File "/Users/keeep/Downloads/py-project/pythonSpider/venv/lib/python3.8/site-packages/aiohttp/streams.py", line 616, in read
    await self._waiter
aiohttp.client_exceptions.ServerDisconnectedError: Server disconnected

发表于 | 分类于

基础使用

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
from parsel import Selector

html = '''
<div>
    <ul>
         <li class="item-0">first item</li>
         <li class="item-1"><a href="link2.html">second item</a></li>
         <li class="item-0 active"><a href="link3.html"><span class="bold">third item</span></a></li>
         <li class="item-1 active"><a href="link4.html">fourth item</a></li>
         <li class="item-0"><a href="link5.html">fifth item</a></li>
     </ul>
 </div>
'''
selector = Selector(text=html)
# 提取文本
items2 = selector.xpath('//li[contains(@class, "item-0")]//text()')
# 提取属性
items3 = selector.xpath('//li[contains(@class, "item-0")]/a/@href')
# 正则提取
result = selector.xpath('//li[contains(@class, "item-0")]/a/@href').re('link.*')
for item in items2:
		# first item
    # third item
    # fifth item
    print(item.get())

发表于 | 分类于

beautifulsoup库

pip install beautifulsoup4

基本使用

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
html = """
<html><head><title>The Dormouse's title</title></head>
<body>
<p class="title" name="dromouse"><b>The Dormouse's story</b></p>
<p class="story">Once upon a time there were three little sisters; and their names were
<a href="http://example.com/elsie" class="sister" id="link1"><!-- Elsie --></a>,
<a href="http://example.com/lacie" class="sister" id="link2">Lacie</a> and
<a href="http://example.com/tillie" class="sister" id="link3">Tillie</a>;
and they lived at the bottom of a well.</p>
<p class="story">...</p>
"""
from bs4 import BeautifulSoup

soup = BeautifulSoup(html, 'lxml')
# <title>The Dormouse's title</title>
print(soup.title)
# The Dormouse's title
print(soup.title.string)
# <head><title>The Dormouse's title</title></head>
print(soup.head)
# The Dormouse's story 当有多个p节点匹配成功时，只会选取第一个
print(soup.p.string)
# dromouse 获取第一个p节点属性名为name的值
print(soup.p.attrs['name'])
print(soup.p['name'])
# html节点下的所有子孙类
for child in soup.html.children:
    print(child)
# a节点的直接父节点
for i in soup.a.parent:
    print(i)
# a节点的所有祖先节点
for i in soup.a.parents:
  	print(i)
# 兄弟节点
print('下一个兄弟节点', soup.a.next_sibling)
print('上一个兄弟节点', soup.a.previous_sibling)

方法选择器

1
2
3
4
5
6
7
8
9
10
11
12
soup = BeautifulSoup(html, 'lxml')
1. find_all
# 查询所有ul节点
print(soup.find_all(name='ul'))
print(soup.find_all(attrs={'id': 'list-1'}))
print(soup.find_all(id='list-1'))
# class是关键字，所以需要加一个下划线区分
print(soup.find_all(class_='element'))
print(soup.find_all(text=re.compile('link')))
2. find
# find返回的是符合条件的单一元素
print(soup.find(name='ul'))

发表于 | 分类于
  • /当前节点的直接子节点
  • //当前节点选取子孙节点
  • .当前节点
  • ..当前节点的父节点
  • @选取属性
  • nodeName选取节点的所有子节点

属性多值匹配

1
2
3
4
5
6
7
from lxml import etree
text = '''
<li class="li li-first" name="item"><a href="link.html">first item</a></li>
'''
html = etree.HTML(text)
# 采用contains函数 意思为class属性中包含li，并且name属性为item的对象
result = html.xpath('//li[contains(@class, "li") and @name="item"]/a/text()')

按序匹配

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
from lxml import etree

text = '''
<div>
    <ul>
         <li class="item-0" name="yyy"><a href="link1.html">first item</a></li>
         <li class="item-1"><a href="link2.html">second item</a></li>
         <li class="item-inactive"><a href="link3.html">third item</a></li>
         <li class="item-1"><a href="link4.html">fourth item</a></li>
         <li class="item-0"><a href="link5.html">fifth item</a>
     </ul>
 </div>
'''
html = etree.HTML(text)
result = html.xpath('//li[1]/a/text()')
result = html.xpath('//li[last()]/a/text()')
# 位置小于3的li节点，也就是1、2节点
result = html.xpath('//li[position()<3]/a/text()')
# 倒数第三个li节点
result = html.xpath('//li[last()-2]/a/text()')

节点轴匹配

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
from lxml import etree

html = etree.HTML(text)
# 返回第一个li节点的所有祖先节点
result = html.xpath('//li[1]/ancestor::*')
# 返回第一个li节点的div祖先节点
result = html.xpath('//li[1]/ancestor::div')
# 返回第一个li节点的所有属性值 ['item-0', 'yyy']
result = html.xpath('//li[1]/attribute::*')
# 返回第一个li节点的所有直接子节点中href属性为link1.html的节点
result = html.xpath('//li[1]/child::a[@href="link1.html"]')
# 返回第一个li节点的所有子孙节点中的span节点，也就是a节点不返回
result = html.xpath('//li[1]/descendant::span')
# 返回第一个li节点之后的所有节点 4个<li> 4个<a>
result = html.xpath('//li[1]/following::*')
# 返回第一个li节点后的所有同级节点 4个<li>
result = html.xpath('//li[1]/following-sibling::*')

发表于 | 分类于

最近买了本《python3网络爬虫开发实战》，之前scrapy爬虫有过研究，但是没有系统的买书研读过，挺感兴趣所以试试

urllib库

urlopen

1
2
3
4
5
6
7
'''
def urlopen(url, data=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
      *, cafile=None, capath=None, cadefault=False, context=None)
'''
# 最基础使用
response = urllib.request.urlopen('https://www.baidu.com')
print(response.read().decode('utf-8'))

Request

1
2
3
4
5
6
import urllib.request
import urllib.parse
# 请求时携带参数
data = bytes(urllib.parse.urlencode({"username": "1"}), encoding='utf-8')
request = urllib.request.Request(url='https://www.baidu.com', data=data, headers={"head": "1"}, method='POST')
urllib.request.urlopen(url=request)

error

1
2
3
4
5
6
from urllib import request, error

try:
    response = request.urlopen('https://cuiqingcai.com/404')
except error.URLError as e:
    print(e.reason)
阅读全文 »

发表于

遇到一个需求，需要Selenium进行POST带参数请求，想到的方法是执行js脚本

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
ChromeDriver webDriver = new ChromeDriver();
String script = "const form = document.createElement('form');\n" +
                "form.method = 'post';\n" +
                "form.action = '%s';\n" +
                " const hiddenField = document.createElement('input');\n" +
                " hiddenField.type = 'hidden';\n" +
                " hiddenField.name = 'pageSizeNow';\n" +
                " hiddenField.value = '10';\n" +
                "\n" +
                " form.appendChild(hiddenField);" +
                " const hiddenField1 = document.createElement('input');\n" +
                " hiddenField1.type = 'hidden';\n" +
                " hiddenField1.name = 'pageIndexNow';\n" +
                " hiddenField1.value = '%s';\n" +
                "\n" +
                " form.appendChild(hiddenField1);" +
                "document.body.appendChild(form);\n" +
                "form.submit();";
 webDriver.executeScript(String.format(script, "https://alk.12348.gov.cn/LawMultiSearch/Search", page.toString()));

发表于

输出顺序：

  1. 同步任务，new Promise()实例
  2. 微任务 Promise.then/catch/finally、process.nextTick
  3. 宏任务 setTimeOut、AJAX、文件操作

发表于

async和await是ES8引入的新语法，用来简化Promise异步操作

用来修饰返回Promise对象的函数

1
2
3
4
5
6
7
// 当函数中出现await，则函数必须用async修饰
async function getAllFile() {
    // 返回Promise对象
    const r2 = thenFs.readFile('code1/files/1.txt', 'utf8')
    // 直接返回内容对象
    const r2 = await thenFs.readFile('code1/files/1.txt', 'utf8')
}
  • 执行顺序 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
// A B C r1 r2 r3 D
console.log('A')
async function getAllFile() {
  console.log('B')
  const r1 = await thenFs.readFile('code1/files/1.txt', 'utf8')
  console.log(r1)
  const r2 = await thenFs.readFile('code1/files/2.txt', 'utf8')
  console.log(r2)
  const r3 = await thenFs.readFile('code1/files/3.txt', 'utf8')
  console.log(r3)
  console.log('D')
}

getAllFile()
console.log('C')
0%