抖音爬取

发表于

兴趣使然，想要尝试爬取抖音视频

技术

  • 基础语言：JAVA
  • 爬虫框架：Webmagic（ps：感觉还是没有python的Scrapy好用）
  • 视频下载：ffmpeg
  • WEB自动化工具：Selenium
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
<dependency>
  <groupId>org.seleniumhq.selenium</groupId>
  <artifactId>selenium-java</artifactId>
  <version>3.141.59</version>
</dependency>
<dependency>
  <groupId>us.codecraft</groupId>
  <artifactId>webmagic-extension</artifactId>
  <version>0.7.3</version>
</dependency>
<dependency>
  <groupId>us.codecraft</groupId>
  <artifactId>webmagic-core</artifactId>
  <version>0.7.3</version>
</dependency>

签名

其他的操作过于简单，这里就不赘述了，此篇博客详细说明是如何获得_signature参数的值

目前版本想要获取抖音的视频列表，需要提供_signature值，正确才能返回视频列表

视频列表格式：https://www.iesdouyin.com/web/api/v2/aweme/post/?sec_uid=MS4wLjABAAAA2JmxeROm5PsNNZ7H1_1U5ek97hQAXxZzbYWUxWMheVs&count=21&max_cursor=0&aid=1128&_signature=1q5zugAAtualWHnVPS-u4Nauc6&dytk=

在一系列参数调适之后，我发现只有3个参数必传（sec_uid、max_cursor、_signature），很好理解，sec_uid为用户的唯一id，max_cursor为视频列表的后缀参数，即想要获取完整的列表需要不断取max_cursor的值进行url更新，_signature为签名

视频列表url精简之后格式：https://www.iesdouyin.com/web/api/v2/aweme/post/?sec_uid=MS4wLjABAAAA2JmxeROm5PsNNZ7H1_1U5ek97hQAXxZzbYWUxWMheVs&max_cursor=0&_signature=cKvA4QAAEN-OIhCGGWpCeXCrwP

  • 第一步：刷新浏览器获取视频列表

  • 第二步：搜索_signature字段

  • 点击之后显示

  • 发现是signature赋值给了_signature，所以继续搜索signature字段，发现下面这一行

  • 继续搜索_bytedAcrawler字段，发现

  • 继续搜索9bd2804c7e68ac461d65,发现了疑似签名的函数，将它全部复制下来

  • 得到结果
1
2
3
4
5
6
7
(function (module, exports) {
    Function(function (t) {
        return 'e(e,a,r){(b[e]||(b[e]=t("x,y","x "+e+" y")(r,a)}a(e,a,r){(k[r]||(k[r]=t("x,y","new x[y]("+Array(r+1).join(",x[y]")(1)+")")(e,a)}r(e,a,r){n,t,s={},b=s.d=r?r.d+1:0;for(s["$"+b]=s,t=0;t<b;t)s[n="$"+t]=r[n];for(t=0,b=s=a;t<b;t)s[t]=a[t];c(e,0,s)}c(t,b,k){u(e){v[x]=e}f{g=,ting(bg)}l{try{y=c(t,b,k)}catch(e){h=e,y=l}}for(h,y,d,g,v=[],x=0;;)switch(g=){case 1:u(!)4:f5:u((e){a=0,r=e;{c=a<r;c&&u(e[a]),c}}(6:y=,u((y8:if(g=,lg,g=,y===c)b+=g;else if(y!==l)y9:c10:u(s(11:y=,u(+y)12:for(y=f,d=[],g=0;g<y;g)d[g]=y.charCodeAt(g)^g+y;u(String.fromCharCode.apply(null,d13:y=,h=delete [y]14:59:u((g=)?(y=x,v.slice(x-=g,y:[])61:u([])62:g=,k[0]=65599*k[0]+k[1].charCodeAt(g)>>>065:h=,y=,[y]=h66:u(e(t[b],,67:y=,d=,u((g=).x===c?r(g.y,y,k):g.apply(d,y68:u(e((g=t[b])<"<"?(b--,f):g+g,,70:u(!1)71:n72:+f73:u(parseInt(f,3675:if(){bcase 74:g=<<16>>16g76:u(k[])77:y=,u([y])78:g=,u(a(v,x-=g+1,g79:g=,u(k["$"+g])81:h=,[f]=h82:u([f])83:h=,k[]=h84:!085:void 086:u(v[x-1])88:h=,y=,h,y89:u({e{r(e.y,arguments,k)}e.y=f,e.x=c,e})90:null91:h93:h=0:;default:u((g<<16>>16)-16)}}n=this,t=n.Function,s=Object.keys||(e){a={},r=0;for(c in e)a[r]=c;a=r,a},b={},k={};r'.replace(/[-]/g, function (m) {
            return t[m.charCodeAt(0) & 15]
        })
    }("v[x++]=v[--x]t.charCodeAt(b++)-32function return ))++.substrvar .length(),b+=;break;case ;break}".split("")))()('gr$Daten Иb/s!l y͒yĹg,(lfi~ah`{mv,-n|jqewVxp{rvmmx,&effkx[!cs"l".Pq%widthl"@q&heightl"vr*getContextx$"2d[!cs#l#,*;?|u.|uc{uq$fontl#vr(fillTextx$$龘ฑภ경2<[#c}l#2q*shadowBlurl#1q-shadowOffsetXl#$$limeq+shadowColorl#vr#arcx88802[%c}l#vr&strokex[ c}l"v,)}eOmyoZB]mx[ cs!0s$l$Pb<k7l l!r&lengthb%^l$1+s$jl  s#i$1ek1s$gr#tack4)zgr#tac$! +0o![#cj?o ]!l$b%s"o ]!l"l$b*b^0d#>>>s!0s%yA0s"l"l!r&lengthb<k+l"^l"1+s"jl  s&l&z0l!$ +["cs\'(0l#i\'1ps9wxb&s() &{s)/s(gr&Stringr,fromCharCodes)0s*yWl ._b&s o!])l l Jb<k$.aj;l .Tb<k$.gj/l .^b<k&i"-4j!+& s+yPo!]+s!l!l Hd>&l!l Bd>&+l!l <d>&+l!l 6d>&+l!l &+ s,y=o!o!]/q"13o!l q"10o!],l 2d>& s.{s-yMo!o!]0q"13o!]*Ld<l 4d#>>>b|s!o!l q"10o!],l!& s/yIo!o!].q"13o!],o!]*Jd<l 6d#>>>b|&o!]+l &+ s0l-l!&l-l!i\'1z141z4b/@d<l"b|&+l-l(l!b^&+l-l&zl\'g,)gk}ejo{cm,)|yn~Lij~em["cl$b%@d<l&zl\'l $ +["cl$b%b|&+l-l%8d<@b|l!b^&+ q$sign ', [Object.defineProperty(exports, '__esModule', {value: !0})]);
})
  • 此时我们回到刚才signature = (0, _bytedAcrawler.sign)(nonce);打断点，看他的值
  • Step之后进入了VM1175文件，显示如下
1
2
3
4
(function anonymous(
) {
function e(e,a,r){return (b[e]||(b[e]=t("x,y","return x "+e+" y")))(r,a)}function a(e,a,r){return (k[r]||(k[r]=t("x,y","return new x[y]("+Array(r+1).join(",x[++y]").substr(1)+")")))(e,a)}function r(e,a,r){var n,t,s={},b=s.d=r?r.d+1:0;for(s["$"+b]=s,t=0;t<b;t++)s[n="$"+t]=r[n];for(t=0,b=s.length=a.length;t<b;t++)s[t]=a[t];return c(e,0,s)}function c(t,b,k){function u(e){v[x++]=e}function f(){return g=t.charCodeAt(b++)-32,t.substring(b,b+=g)}function l(){try{y=c(t,b,k)}catch(e){h=e,y=l}}for(var h,y,d,g,v=[],x=0;;)switch(g=t.charCodeAt(b++)-32){case 1:u(!v[--x]);break;case 4:v[x++]=f();break;case 5:u(function (e){var a=0,r=e.length;return function (){var c=a<r;return c&&u(e[a++]),c}}(v[--x]));break;case 6:y=v[--x],u(v[--x](y));break;case 8:if(g=t.charCodeAt(b++)-32,l(),b+=g,g=t.charCodeAt(b++)-32,y===c)b+=g;else if(y!==l)return y;break;case 9:v[x++]=c;break;case 10:u(s(v[--x]));break;case 11:y=v[--x],u(v[--x]+y);break;case 12:for(y=f(),d=[],g=0;g<y.length;g++)d[g]=y.charCodeAt(g)^g+y.length;u(String.fromCharCode.apply(null,d));break;case 13:y=v[--x],h=delete v[--x][y];break;case 14:v[x++]=t.charCodeAt(b++)-32;break;case 59:u((g=t.charCodeAt(b++)-32)?(y=x,v.slice(x-=g,y)):[]);break;case 61:u(v[--x][t.charCodeAt(b++)-32]);break;case 62:g=v[--x],k[0]=65599*k[0]+k[1].charCodeAt(g)>>>0;break;case 65:h=v[--x],y=v[--x],v[--x][y]=h;break;case 66:u(e(t[b++],v[--x],v[--x]));break;case 67:y=v[--x],d=v[--x],u((g=v[--x]).x===c?r(g.y,y,k):g.apply(d,y));break;case 68:u(e((g=t[b++])<"<"?(b--,f()):g+g,v[--x],v[--x]));break;case 70:u(!1);break;case 71:v[x++]=n;break;case 72:v[x++]=+f();break;case 73:u(parseInt(f(),36));break;case 75:if(v[--x]){b++;break}case 74:g=t.charCodeAt(b++)-32<<16>>16,b+=g;break;case 76:u(k[t.charCodeAt(b++)-32]);break;case 77:y=v[--x],u(v[--x][y]);break;case 78:g=t.charCodeAt(b++)-32,u(a(v,x-=g+1,g));break;case 79:g=t.charCodeAt(b++)-32,u(k["$"+g]);break;case 81:h=v[--x],v[--x][f()]=h;break;case 82:u(v[--x][f()]);break;case 83:h=v[--x],k[t.charCodeAt(b++)-32]=h;break;case 84:v[x++]=!0;break;case 85:v[x++]=void 0;break;case 86:u(v[x-1]);break;case 88:h=v[--x],y=v[--x],v[x++]=h,v[x++]=y;break;case 89:u(function (){function e(){return r(e.y,arguments,k)}return e.y=f(),e.x=c,e}());break;case 90:v[x++]=null;break;case 91:v[x++]=h;break;case 93:h=v[--x];break;case 0:return v[--x];default:u((g<<16>>16)-16)}}var n=this,t=n.Function,s=Object.keys||function (e){var a={},r=0;for(var c in e)a[r++]=c;return a.length=r,a},b={},k={};return r
})
  • 将两者整合，生成最终的sign.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
function getSignature() {
    var t = {};
  	// 经过调试，不加此参数报错，日志为缺少userAgent参数
    this.navigator = {
        "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.114 Safari/537.36"
    }
    // 立即调用函数 这里为上面VM1175文件内的函数
    !function () {
        function e(e, a, r) {
            return (b[e] || (b[e] = t("x,y", "return x " + e + " y")))(r, a)
        }

        function a(e, a, r) {
            return (k[r] || (k[r] = t("x,y", "return new x[y](" + Array(r + 1).join(",x[++y]").substr(1) + ")")))(e, a)
        }

        function r(e, a, r) {
            var n, t, s = {}, b = s.d = r ? r.d + 1 : 0;
            for (s["$" + b] = s, t = 0; t < b; t++) s[n = "$" + t] = r[n];
            for (t = 0, b = s.length = a.length; t < b; t++) s[t] = a[t];
            return c(e, 0, s)
        }

        function c(t, b, k) {
            function u(e) {
                v[x++] = e
            }

            function f() {
                return g = t.charCodeAt(b++) - 32, t.substring(b, b += g)
            }

            function l() {
                try {
                    y = c(t, b, k)
                } catch (e) {
                    h = e, y = l
                }
            }

            for (var h, y, d, g, v = [], x = 0; ;) switch (g = t.charCodeAt(b++) - 32) {
                case 1:
                    u(!v[--x]);
                    break;
                case 4:
                    v[x++] = f();
                    break;
                case 5:
                    u(function (e) {
                        var a = 0, r = e.length;
                        return function () {
                            var c = a < r;
                            return c && u(e[a++]), c
                        }
                    }(v[--x]));
                    break;
                case 6:
                    y = v[--x], u(v[--x](y));
                    break;
                case 8:
                    if (g = t.charCodeAt(b++) - 32, l(), b += g, g = t.charCodeAt(b++) - 32, y === c) b += g; else if (y !== l) return y;
                    break;
                case 9:
                    v[x++] = c;
                    break;
                case 10:
                    u(s(v[--x]));
                    break;
                case 11:
                    y = v[--x], u(v[--x] + y);
                    break;
                case 12:
                    for (y = f(), d = [], g = 0; g < y.length; g++) d[g] = y.charCodeAt(g) ^ g + y.length;
                    u(String.fromCharCode.apply(null, d));
                    break;
                case 13:
                    y = v[--x], h = delete v[--x][y];
                    break;
                case 14:
                    v[x++] = t.charCodeAt(b++) - 32;
                    break;
                case 59:
                    u((g = t.charCodeAt(b++) - 32) ? (y = x, v.slice(x -= g, y)) : []);
                    break;
                case 61:
                    u(v[--x][t.charCodeAt(b++) - 32]);
                    break;
                case 62:
                    g = v[--x], k[0] = 65599 * k[0] + k[1].charCodeAt(g) >>> 0;
                    break;
                case 65:
                    h = v[--x], y = v[--x], v[--x][y] = h;
                    break;
                case 66:
                    u(e(t[b++], v[--x], v[--x]));
                    break;
                case 67:
                    y = v[--x], d = v[--x], u((g = v[--x]).x === c ? r(g.y, y, k) : g.apply(d, y));
                    break;
                case 68:
                    u(e((g = t[b++]) < "<" ? (b--, f()) : g + g, v[--x], v[--x]));
                    break;
                case 70:
                    u(!1);
                    break;
                case 71:
                    v[x++] = n;
                    break;
                case 72:
                    v[x++] = +f();
                    break;
                case 73:
                    u(parseInt(f(), 36));
                    break;
                case 75:
                    if (v[--x]) {
                        b++;
                        break
                    }
                case 74:
                    g = t.charCodeAt(b++) - 32 << 16 >> 16, b += g;
                    break;
                case 76:
                    u(k[t.charCodeAt(b++) - 32]);
                    break;
                case 77:
                    y = v[--x], u(v[--x][y]);
                    break;
                case 78:
                    g = t.charCodeAt(b++) - 32, u(a(v, x -= g + 1, g));
                    break;
                case 79:
                    g = t.charCodeAt(b++) - 32, u(k["$" + g]);
                    break;
                case 81:
                    h = v[--x], v[--x][f()] = h;
                    break;
                case 82:
                    u(v[--x][f()]);
                    break;
                case 83:
                    h = v[--x], k[t.charCodeAt(b++) - 32] = h;
                    break;
                case 84:
                    v[x++] = !0;
                    break;
                case 85:
                    v[x++] = void 0;
                    break;
                case 86:
                    u(v[x - 1]);
                    break;
                case 88:
                    h = v[--x], y = v[--x], v[x++] = h, v[x++] = y;
                    break;
                case 89:
                    u(function () {
                        function e() {
                            return r(e.y, arguments, k)
                        }

                        return e.y = f(), e.x = c, e
                    }());
                    break;
                case 90:
                    v[x++] = null;
                    break;
                case 91:
                    v[x++] = h;
                    break;
                case 93:
                    h = v[--x];
                    break;
                case 0:
                    return v[--x];
                default:
                    u((g << 16 >> 16) - 16)
            }
        }

        var n = this, t = n.Function, s = Object.keys || function (e) {
            var a = {}, r = 0;
            for (var c in e) a[r++] = c;
            return a.length = r, a
        }, b = {}, k = {};
        return r
    }()
  // 这里为9bd2804c7e68ac461d65混淆的函数，直接放在这
    ('gr$Daten Иb/s!l y͒yĹg,(lfi~ah`{mv,-n|jqewVxp{rvmmx,&effkx[!cs"l".Pq%widthl"@q&heightl"vr*getContextx$"2d[!cs#l#,*;?|u.|uc{uq$fontl#vr(fillTextx$$龘ฑภ경2<[#c}l#2q*shadowBlurl#1q-shadowOffsetXl#$$limeq+shadowColorl#vr#arcx88802[%c}l#vr&strokex[ c}l"v,)}eOmyoZB]mx[ cs!0s$l$Pb<k7l l!r&lengthb%^l$1+s$jl  s#i$1ek1s$gr#tack4)zgr#tac$! +0o![#cj?o ]!l$b%s"o ]!l"l$b*b^0d#>>>s!0s%yA0s"l"l!r&lengthb<k+l"^l"1+s"jl  s&l&z0l!$ +["cs\'(0l#i\'1ps9wxb&s() &{s)/s(gr&Stringr,fromCharCodes)0s*yWl ._b&s o!])l l Jb<k$.aj;l .Tb<k$.gj/l .^b<k&i"-4j!+& s+yPo!]+s!l!l Hd>&l!l Bd>&+l!l <d>&+l!l 6d>&+l!l &+ s,y=o!o!]/q"13o!l q"10o!],l 2d>& s.{s-yMo!o!]0q"13o!]*Ld<l 4d#>>>b|s!o!l q"10o!],l!& s/yIo!o!].q"13o!],o!]*Jd<l 6d#>>>b|&o!]+l &+ s0l-l!&l-l!i\'1z141z4b/@d<l"b|&+l-l(l!b^&+l-l&zl\'g,)gk}ejo{cm,)|yn~Lij~em["cl$b%@d<l&zl\'l $ +["cl$b%b|&+l-l%8d<@b|l!b^&+ q$sign ', 
     // 这里需要将exports换成定义的签名t
     [Object.defineProperty(t, '__esModule', {
        value: !0
    })])
      return t.sign();
}

这里比较坑的地方出现了，当js引用在html中，console显示出的sign值是正确的，如果直接用java直接调用js生成的sign值则是错误的，所以猜测抖音用了什么技术使签名的生成必须通过浏览器，所以最终我采取了用selenium来模拟浏览器生成sign

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
/**
  * 获取抖音签名
  */
public static String getSign() throws InterruptedException {
  System.setProperty("webdriver.chrome.driver", "/usr/local/bin/chromedriver");
  ChromeOptions chromeOptions = new ChromeOptions();
  // 这里无头浏览策略，没有必要频繁打开浏览器
  chromeOptions.addArguments("--headless");
  ChromeDriver driver = new ChromeDriver(chromeOptions);
  driver.get("file:///Users/mintaoyu/projectList/webmagic/src/main/java/sign.html");
  ((JavascriptExecutor) driver).executeScript("document.getElementById(\"val\").value = getSignature();");
  Thread.sleep(1000L);
  String sign = driver.findElementById("val").getAttribute("value");
  driver.quit();
  return sign;
}

最终结果

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
/**
 * 抖音
 * 针对某个用户的全部视频爬取
 */
public class DouYinProcessor implements PageProcessor {

    /**
     * 部分一：抓取网站的相关配置，包括编码、抓取间隔、重试次数等
     */
    private final Site site = Site.me().setRetryTimes(3).setSleepTime(1000).setTimeOut(10000).setUserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.114 Safari/537.36").addHeader("accept-language", "zh-CN,zh;q=0.9,en;q=0.8");

    private final static String DOMAIN = "https://v.douyin.com/eM5wkdb/";

    private final String detailRegex = "https://www.iesdouyin.com/share/user/*";

    @SneakyThrows
    @Override
    public void process(Page page) {
        String url = page.getUrl().toString();
        String secUid = getParam(url, "sec_uid");
        if (page.getUrl().regex(detailRegex).match()) {
            JSONObject userInfo = JSON.parseObject(HttpUtil.get("https://www.iesdouyin.com/web/api/v2/user/info/?sec_uid=" + secUid));
            String nickname = userInfo.getJSONObject("user_info").getString("nickname");
            RuntimeUtil.execForStr("mkdir /Users/mintaoyu/movie/" + nickname);
            String sign = getSign();
            String movieUrl = "https://www.iesdouyin.com/web/api/v2/aweme/post/?sec_uid=" + secUid + "&max_cursor=" + "0" + "&_signature=" + sign;
            page.addTargetRequest(movieUrl);
        } else {
            String sign = getParam(url, "_signature");
            JSONObject rawText = JSON.parseObject(page.getRawText());
            Long maxCursor = rawText.getLong("max_cursor");
            JSONArray awemeList = rawText.getJSONArray("aweme_list");
            if (awemeList.size() == 0) {
                page.setSkip(true);
            }
            for (int i = 0; i < awemeList.size(); i++) {
                String movieUrl = awemeList.getJSONObject(i).getJSONObject("video").getJSONObject("play_addr_lowbr").getJSONArray("url_list").getString(0);
                String desc = awemeList.getJSONObject(i).getString("desc");
                String nickname = awemeList.getJSONObject(i).getJSONObject("author").getString("nickname");
                if (StrUtil.isBlank(desc)) {
                    desc = RandomUtil.randomNumbers(7);
                }
                String cmd = "/usr/local/Cellar/ffmpeg/4.2.2_1/bin/ffmpeg -y -i  '" + movieUrl + "' -acodec copy -vcodec copy ~/movie/" + nickname + "/'" + desc + ".mp4'";
                Process process = Runtime.getRuntime().exec(new String[]{"sh", "-c", cmd});
                RuntimeUtil.getResultLines(process);
                System.out.println("完成" + desc + ".mp4");
            }
            page.addTargetRequest("https://www.iesdouyin.com/web/api/v2/aweme/post/?sec_uid=" + secUid + "&max_cursor=" + maxCursor + "&_signature=" + sign);
        }
    }


    @Override
    public Site getSite() {
        return site;
    }

  	/**
     * 获取重定向url
     */
    public static String getRedirectUrl(String url) {
        CloseableHttpClient httpclient = HttpClients.createDefault();
        HttpClientContext context = HttpClientContext.create();
        HttpGet httpget = new HttpGet(url);
        CloseableHttpResponse response = null;
        String absUrl = null;
        try {
            response = httpclient.execute(httpget, context);
            HttpHost target = context.getTargetHost();
            List<URI> redirectLocations = context.getRedirectLocations();
            URI location = URIUtils.resolve(httpget.getURI(), target, redirectLocations);
            System.out.println("Final HTTP location: " + location.toASCIIString());
            absUrl = location.toASCIIString();
        } catch (IOException e) {
            e.printStackTrace();
        } catch (URISyntaxException e) {
            e.printStackTrace();
        } finally {
            try {
                httpclient.close();
                response.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
        return absUrl;
    }

    /**
     * 获取抖音签名
     */
    public static String getSign() throws InterruptedException {
        System.setProperty("webdriver.chrome.driver", "/usr/local/bin/chromedriver");
        ChromeOptions chromeOptions = new ChromeOptions();
        chromeOptions.addArguments("--headless");
        ChromeDriver driver = new ChromeDriver(chromeOptions);
        driver.get("file:///Users/mintaoyu/projectList/webmagic/src/main/java/sign.html");
        ((JavascriptExecutor) driver).executeScript("document.getElementById(\"val\").value = getSignature();");
        Thread.sleep(1000L);
        String sign = driver.findElementById("val").getAttribute("value");
        driver.quit();
        return sign;
    }

    /**
     * 获取url参数
     */
    public static String getParam(String url, String name) {
        String params = url.substring(url.indexOf("?") + 1);
        Map<String, String> split = Splitter.on("&").withKeyValueSeparator("=").split(params);
        return split.get(name);
    }

    public static void main(String[] args) {
        Spider
                .create(new DouYinProcessor())
                .addUrl(getRedirectUrl(DOMAIN))
                .addPipeline(new ConsolePipeline())
                .thread(5)
                .run();
    }
}

缺点

  • 现在是通过手机端抖音APP找到想要下载的对象，分享之后，设置domain，比较麻烦
  • 一些翻转的视频没有办法识别出来利用ffmpeg去旋转(尝试过ws.schild包，他的逻辑是根据视频的长宽来进行判断，比如宽大于长则说明视频是横着的，反之则是竖着的，但是抖音视频不适用，不管横竖尺寸一样)
赏个🍗吧
0%