发表于

因为初始设置只有4G的内存，大一些的APP反混淆的时候会内存溢出，所以这里需要设置一下JVM大小

  • 本次案例由brew安装
  • 路径为/opt/homebrew/Cellar/jadx/1.3.3/libexec/bin/jadx-gui

其他jadx-gui皆没用，只有设置这个路径下的才行

1
DEFAULT_JVM_OPTS='"-Xms1024M" "-Xmx8g"'

发表于 | 分类于

几年之前有总结过Scrapy框架，这次重新系统学习，补充一些不足

Scrapy结合ES

  • 第一步创建es bean
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
# pip install elasticsearch_dsl
from elasticsearch_dsl import Document, Text, Keyword
from elasticsearch_dsl.connections import connections

# 新建连接
connections.create_connection(hosts="http://192.168.0.158:9200")

class Domain(Document):
    # 文章类型
    text = Text(analyzer="ik_max_word")
    author = Keyword()
    tags = Text(analyzer="ik_max_word")

    class Index:
        # 数据库名称和表名称
        name = "scrapy"
        
# 这一步是必须的，没有则分词不生效
if __name__ == '__main__':
    Domain.init()
  • 第二步在Pipeline中将值给附上，并设置中开启该Pipeline
1
2
3
4
5
6
7
8
9
10
from scrapytoscrape.scrapyto.bean import Domain

class ScrapytoscrapePipeline:
    def process_item(self, item, spider):
        domain = Domain()
        domain['text'] = item['text']
        domain['author'] = item['author']
        domain['tags'] = item['tags']
        domain.save()
        return item
阅读全文 »

发表于 | 分类于

npm i express

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
const CryptoJS = require('./crypto.js')
const express = require('express')
const app = express()
const port = 3000
app.use(express.json())

function getToken(player) {
    let key = CryptoJS.enc.Utf8.parse("fipFfVsZsTda94hJNKJfLoaqyqMZFFimwLt");
    const {name, birthday, height, weight} = player;
    let base64Name = CryptoJS.enc.Base64.stringify(CryptoJS.enc.Utf8.parse(name));
    let encrypted = CryptoJS.DES.encrypt(
        `${base64Name}${birthday}${height}${weight}`,
        key,
        {
            mode: CryptoJS.mode.ECB,
            padding: CryptoJS.pad.Pkcs7,
        }
    );
    return encrypted.toString();
}

app.post("/", (req, res) => {
        const data = req.body
        res.send(getToken(data))
    }
)

app.listen(port, () => {
})
1
2
3
4
5
6
7
8
9
10
11
12
import requests

item = {
    'name': '凯文-杜兰特',
    'image': 'durant.png',
    'birthday': '1988-09-29',
    'height': '208cm',
    'weight': '108.9KG'
}

response = requests.post('http://localhost:3000', json=item)
print(response.text)

发表于

准备材料

getToken.js

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
const players = [
  {
    name: '凯文-杜兰特',
    image: 'durant.png',
    birthday: '1988-09-29',
    height: '208cm',
    weight: '108.9KG'
  }, {
    name: '勒布朗-詹姆斯',
    image: 'james.png',
    birthday: '1984-12-30',
    height: '206cm',
    weight: '113.4KG'
  }
]
new Vue({
  el: '#app',
  data: function () {
    return {
      players,
      key: 'fipFfVsZsTda94hJNKJfLoaqyqMZFFimwLt'
    }
  },
  methods: {
    getToken(player) {
      let key = CryptoJS.enc.Utf8.parse(this.key)
      const {name, birthday, height, weight} = player
      // 引用CryptoJS
      let base64Name = CryptoJS.enc.Base64.stringify(CryptoJS.enc.Utf8.parse(name))
      // 引用CryptoJS
      let encrypted = CryptoJS.DES.encrypt(`${base64Name}${birthday}${height}${weight}`, key, {
        // 引用CryptoJS
        mode: CryptoJS.mode.ECB,
        padding: CryptoJS.pad.Pkcs7
      })
      return encrypted.toString()
    }
  }
})
阅读全文 »

发表于 | 分类于

Cookie和Session

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
# 方式一
# requests会自动保存每次请求后设置的Cookie，下次请求自动带上
session = requests.Session()
session.post(LOGIN_URL, data={
    'username': USERNAME,
    'password': PASSWORD
})
response_index = session.get(INDEX_URL)

# 方式二
# 采用Selenium模拟登陆保存Cookie
browser = webdriver.Chrome()
browser.get(BASE_URL)
browser.find_element_by_css_selector('input[name="username"]').send_keys(USERNAME)
browser.find_element_by_css_selector('input[name="password"]').send_keys(PASSWORD)
browser.find_element_by_css_selector('input[type="submit"]').click()
time.sleep(10)
cookies = browser.get_cookies()
browser.close()
session = requests.Session()
# 唯一需要注意的是：Selenium获取的Cookie需要设置进session中
for cookie in cookies:
    session.cookies.set(cookie['name'], cookie['value'])
response_index = session.get(INDEX_URL)

JWT登陆

1
2
3
4
5
6
7
8
9
10
11
12
session = requests.Session()

response_login = session.post(LOGIN_URL, data={
    'username': USERNAME,
    'password': PASSWORD
})

jwt = response_login.json().get('token')
headers = {
    'Authorization': f'jwt {jwt}'
}
response = requests.get(INDEX_URL, params={'limit': 18, 'offset': 0}, headers=headers)

发表于 | 分类于

OCR识别

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
brew install imagemagick
brew install tesseract-lang
pip3 install tesserocr pillow

# 普通无干扰情况下的识别
import tesserocr

print(tesserocr.file_to_text('code.jpg'))
# 有多余点干扰的识别（其中干扰点的颜色比文字浅，比如文字黑色，干扰点彩色）
import tesserocr
from PIL import Image
import numpy as np

image = Image.open('captcha2.png') #图一
# 将图片由彩色转为灰度图像
image = image.convert('L') # 图二
# 设定灰度阈值 自设置根据实际情况设置阈值去除噪声
threshold = 50
# 将图片转为Numpy数组
array = np.array(image)
# 通过where方法对数组进行筛选
# 将灰度大于阈值的图片像素设置为255，也就是白色，否则设置为0，也就是黑色(去噪处理)
array = np.where(array > threshold, 255, 0)
image = Image.fromarray(array.astype('uint8')) # 图三
print(tesserocr.image_to_text(image))

一些深度学习的办法目前无法理解，之后了解完深度学习再回来补充

发表于 | 分类于

CSS位置偏移反爬案例

1
2
3
4
5
6
7
8
9
10
11
12
13
<!--可以看到此处的书名分为了多个span节点，并且顺序打乱-->
<h3 data-v-7f1a77ef="" class="m-b-sm name">
  <span data-v-7f1a77ef="" class="char" style="left: 16px;">然</span>
  <span data-v-7f1a77ef="" class="char" style="left: 48px;">动</span>
  <span data-v-7f1a77ef="" class="char" style="left: 0px;">怦</span>
  <span data-v-7f1a77ef="" class="char" style="left: 32px;">心</span>
</h3>
<!--可以看到position属性为absolute，代表绝对定位-->
<!--根据left值进行排序后渲染到页面上-->
.name .char[data-v-7f1a77ef] {
    display: inline-block;
    position: absolute;
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
from selenium import webdriver
from selenium.common.exceptions import TimeoutException
from selenium.webdriver import ChromeOptions
from selenium.webdriver.common.by import By
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import logging
import re

logging.basicConfig(level=logging.INFO,
                    format='%(asctime)s - %(levelname)s: %(message)s')

option = ChromeOptions()
option.add_experimental_option('excludeSwitches', ['enable-automation'])
option.add_experimental_option('useAutomationExtension', False)
browser = webdriver.Chrome(options=option, executable_path='/Users/keeep/Downloads/tool/chromedriver')
browser.execute_cdp_cmd('Page.addScriptToEvaluateOnNewDocument', {
    'source': 'Object.defineProperty(navigator, "webdriver", {get: () => undefined})'
})
wait = WebDriverWait(browser, 10)
URL_PAGE = 'https://antispider3.scrape.center/'


def get_page():
    try:
        browser.get(URL_PAGE)
        # 为了能够获取有效信息，这里采用visibility_of_element_located获取
        # 可见性意味着元素不仅被显示 但也有一个大于 0 的高度和宽度
        wait.until(EC.visibility_of_element_located(locator=(By.XPATH, '//h3[contains(@class,"name")]/span')))
    except TimeoutException:
        logging.error('error occurred while scraping %s', URL_PAGE, exc_info=True)


def get_title():
    info_list = []
    elements = browser.find_elements(By.XPATH, '//h3[contains(@class,"name")]')
    for element in elements:
        element_find_elements = element.find_elements(By.XPATH, './span')
        info = {}
        if len(element_find_elements) != 0:
            for span_ele in element_find_elements:
                ele_text = span_ele.text
                if len(ele_text) != 0:
                    style = span_ele.get_attribute('style')
                    search = int(re.search('\\d+', style).group())
                    info[search] = ele_text
            keys = sorted(info.keys())
            output_list = [''.join([info[key] for key in keys])]
            info_list.append(output_list)
        else:
            info_list.append(element.text)


if __name__ == '__main__':
    try:
        get_page()
        get_title()
    finally:
        browser.close()

字体内容放置在CSS中案例

1
2
3
4
5
<p data-v-090744c8="" class="score m-t-md m-b-n-sm">
    <span data-v-090744c8=""><i data-v-090744c8="" class="icon icon-789">::before</i></span>
    <span data-v-090744c8=""><i data-v-090744c8="" class="icon icon-981">::before</i></span>
    <span data-v-090744c8=""><i data-v-090744c8="" class="icon icon-504">::before</i></span>
</p>
1
2
3
4
5
6
7
8
9
10
11
.icon-504:before {
    content: "8"
}

.icon-789:before {
    content: "9"
}

.icon-981:before {
    content: "."
}

所以最后渲染出结果为9.8由上面可以发现前端将数据隐藏在了CSS文件中，找到该css文件进行请求

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
import requests
from selenium import webdriver
from selenium.common.exceptions import TimeoutException
from selenium.webdriver import ChromeOptions
from selenium.webdriver.common.by import By
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import logging
import re

logging.basicConfig(level=logging.INFO,
                    format='%(asctime)s - %(levelname)s: %(message)s')
option = ChromeOptions()
option.add_experimental_option('excludeSwitches', ['enable-automation'])
option.add_experimental_option('useAutomationExtension', False)
browser = webdriver.Chrome(options=option, executable_path='/Users/keeep/Downloads/tool/chromedriver')
browser.execute_cdp_cmd('Page.addScriptToEvaluateOnNewDocument', {
    'source': 'Object.defineProperty(navigator, "webdriver", {get: () => undefined})'
})
wait = WebDriverWait(browser, 10)
RE_RULE = '.icon-(.*?):before\{content:"(.*?)"\}'
URL_PAGE = 'https://antispider4.scrape.center/'
response = requests.get('https://antispider4.scrape.center/css/app.654ba59e.css')
pattern = re.compile(RE_RULE)
findall = re.findall(pattern, response.text)
# 获取字典
info = {list_result[0]: list_result[1] for list_result in findall}


def get_page():
    try:
        browser.get(URL_PAGE)
        wait.until(EC.visibility_of_element_located(locator=(By.XPATH, '//p[contains(@class,"score")]')))
    except TimeoutException:
        logging.error('error occurred while scraping %s', URL_PAGE, exc_info=True)


def get_info():
    elements = browser.find_elements(By.XPATH, '//div[contains(@class,"item")]')
    for element in elements:
      	# 在前一个元素获取的情况下获取其子类
        element.find_element(By.XPATH, './/a[@class="name"]/h2')
        title = element.text
        score = get_score(element)
        print(title, score)


def get_score(element):
    global join
    ele_find_elements = element.find_elements(By.XPATH, './/p[contains(@class,"score")]/span/i')
    score = []
    for find_element in ele_find_elements:
        class_attr = find_element.get_attribute('class')
        re_class = re.search('icon-(\d+)', class_attr).group(1)
        info_get = info.get(re_class)
        score.append(info_get)
    join = "".join(score)
    return join

发表于 | 分类于

代码生成

1
2
3
4
5
6
# -o 代表输出名称
# --target 使用语言，默认为python同步，如需异步使用python-async
# -b 代表用什么浏览器，默认chromium
# --device 模拟手机浏览器
# --lang 浏览器语言
playwright codegen -o my_script.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
from playwright.sync_api import sync_playwright

def on_response(response):
    if '/api/movie/' in response.url and response.status == 200:
        print(response.json())

# 用这个方法监听请求，即使请求中包含加密参数也不要紧，因为这儿直接截获的是最后的响应结果
with sync_playwright() as p:
    browser = p.chromium.launch(headless=False)
    page = browser.new_page()
    # 监听response事件
    page.on('response', on_response)
    page.goto('https://spa6.scrape.center/')
    page.wait_for_load_state('networkidle')
    browser.close()

发表于 | 分类于 
1
2
3
4
5
6
7
8
9
10
-- Splash可以采用lua脚本的形式去渲染
function main(splash, args)
  assert(splash:go(args.url))
  assert(splash:wait(0.5))
  return {
    html = splash:html(),
    png = splash:png(),
    har = splash:har(),
  }
end

API

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
# render.html获取js渲染页面的html代码
url = 'http://localhost:8050/render.html?url=https://www.baidu.com&amp;wait=5'
# render.png/render.jpeg 返回页面截图
# render.har 返回页面加载过程信息
# render.json 可包含前面3种信息 需自行添加html=1，har=1，png=1等等
url = 'http://localhost:8050/render.json?url=https://www.baidu.com&html=1&har=1&png=1'
response = requests.get(url).json()
# 获取的图片是base64需要转换
imgdata = base64.b64decode(response.get('png'))
with open('x.png', 'wb') as f:
    f.write(imgdata)
# execute 实现于Lua脚本的对接
lua = '''
function main(splash, args)
  assert(splash:go('https://baidu.com'))
  return {
    html = splash:html(),
    png = splash:png(),
    har = splash:har(),
  }
end
'''

url = 'http://localhost:8050/execute?lua_source=' + quote(lua)
response = requests.get(url)
print(response.text)
0%