公开页抓取改为 Chrome 136 自洽身份 + 每节点 SQLite 养罐,Trafilatura 走 curl_cffi;MCP/README/接手说明与 09-02 现网复测对齐,避免消费方继续抄过期的站点三分表。 Co-authored-by: Cursor <cursoragent@cursor.com>
75 lines
2.1 KiB
Go
75 lines
2.1 KiB
Go
package dock
|
||
|
||
import (
|
||
"encoding/json"
|
||
"time"
|
||
|
||
"onesvm.com/onesvm/browser-server/internal/contract"
|
||
)
|
||
|
||
// cdpApplySession UA + Cookie 注入(失败不致命:Lightpanda 可能缺 Network.setCookies)。
|
||
func cdpApplySession(send func(string, any, time.Duration) (json.RawMessage, error), sess *contract.SessionAttach) {
|
||
ua := cdpUA
|
||
if sess != nil && sess.UserAgent != "" {
|
||
ua = sess.UserAgent
|
||
}
|
||
_, _ = send("Network.setUserAgentOverride", map[string]any{"userAgent": ua}, 8*time.Second)
|
||
if sess == nil || len(sess.Cookies) == 0 {
|
||
return
|
||
}
|
||
payload := make([]map[string]any, 0, len(sess.Cookies))
|
||
for _, c := range sess.Cookies {
|
||
if c.Name == "" || c.Value == "" {
|
||
continue
|
||
}
|
||
item := map[string]any{"name": c.Name, "value": c.Value, "path": c.Path}
|
||
if item["path"] == "" {
|
||
item["path"] = "/"
|
||
}
|
||
if c.Domain != "" {
|
||
item["domain"] = c.Domain
|
||
} else if sess.ETLD != "" {
|
||
item["domain"] = sess.ETLD
|
||
}
|
||
payload = append(payload, item)
|
||
}
|
||
if len(payload) == 0 {
|
||
return
|
||
}
|
||
_, _ = send("Network.setCookies", map[string]any{"cookies": payload}, 8*time.Second)
|
||
}
|
||
|
||
// cdpCollectCookies 拉取当前页一等 Cookie(失败返回 nil)。
|
||
func cdpCollectCookies(send func(string, any, time.Duration) (json.RawMessage, error), pageURL string) []contract.Cookie {
|
||
raw, err := send("Network.getCookies", map[string]any{"urls": []string{pageURL}}, 5*time.Second)
|
||
if err != nil || len(raw) == 0 {
|
||
return nil
|
||
}
|
||
var wrap struct {
|
||
Cookies []struct {
|
||
Name string `json:"name"`
|
||
Value string `json:"value"`
|
||
Domain string `json:"domain"`
|
||
Path string `json:"path"`
|
||
Expires float64 `json:"expires"`
|
||
} `json:"cookies"`
|
||
}
|
||
if json.Unmarshal(raw, &wrap) != nil {
|
||
return nil
|
||
}
|
||
out := make([]contract.Cookie, 0, len(wrap.Cookies))
|
||
for _, c := range wrap.Cookies {
|
||
if c.Name == "" || c.Value == "" {
|
||
continue
|
||
}
|
||
item := contract.Cookie{Name: c.Name, Value: c.Value, Domain: c.Domain, Path: c.Path}
|
||
if c.Expires > 0 {
|
||
item.Expires = int64(c.Expires)
|
||
}
|
||
out = append(out, item)
|
||
if len(out) >= 20 {
|
||
break
|
||
}
|
||
}
|
||
return out
|
||
}
|