本文目录导读：
我来详细说明Python爬虫提取JavaScript变量的几种方法：
正则表达式提取（简单场景）
适用于直接出现在HTML页面中的JS变量：
import re
import requests
def extract_js_vars_regex(url):
response = requests.get(url)
html = response.text
# 提取变量值
patterns = {
'token': r'var\s+token\s*=\s*["\']([^"\']+)["\']',
'config': r'var\s+config\s*=\s*({[^}]+})',
'list': r'var\s+dataList\s*=\s*(\[[^\]]+\])'
}
results = {}
for name, pattern in patterns.items():
match = re.search(pattern, html, re.IGNORECASE)
if match:
results[name] = match.group(1)
return results
使用PyMiniRacer（推荐）
提取并执行JavaScript代码：
from py_mini_racer import MiniRacer
import requests
def extract_js_vars_execute(url):
response = requests.get(url)
html = response.text
# 创建JS执行环境
ctx = MiniRacer()
# 执行整个页面JS
ctx.eval(html)
# 提取变量
try:
token = ctx.eval("token")
config = ctx.eval("config")
return {
'token': token,
'config': config
}
except:
return None
使用Selenium（复杂场景）
适用于动态渲染的页面：
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
import json
def extract_js_vars_selenium(url):
options = Options()
options.add_argument('--headless')
driver = webdriver.Chrome(options=options)
driver.get(url)
# 执行JS获取变量
js_code = """
return {
'window': {
'token': window.token,
'config': window.config,
'dataList': window.dataList
},
'localStorage': {
'token': localStorage.getItem('token'),
'config': localStorage.getItem('config')
}
};
"""
result = driver.execute_script(js_code)
driver.quit()
return result
解析JSON对象（常见场景）
import json
import re
import requests
def extract_json_vars(url):
response = requests.get(url)
html = response.text
# 寻找JSON变量
patterns = [
r'var\s+\w+\s*=\s*(\{.*?\});',
r'window\.\w+\s*=\s*(\{.*?\});',
r'JSON\.parse\(["\'](.+?)["\']\)'
]
results = []
for pattern in patterns:
matches = re.finditer(pattern, html, re.DOTALL)
for match in matchs:
try:
# 尝试解析JSON
data = json.loads(match.group(1))
results.append(data)
except:
continue
return results
提取多级嵌套变量
from py_mini_racer import MiniRacer
import requests
class JsExtractor:
def __init__(self):
self.ctx = MiniRacer()
def extract_variables(self, url, var_paths):
"""
var_paths: 要提取的变量路径列表
如 ['token', 'config.apiKey', 'user.info.name']
"""
response = requests.get(url)
self.ctx.eval(response.text)
results = {}
for path in var_paths:
try:
value = self.ctx.eval(path)
results[path] = value
except:
results[path] = None
return results
# 使用示例
extractor = JsExtractor()
vars_to_extract = ['token', 'config', 'dataList[0].id']
result = extractor.extract_variables('https://example.com', vars_to_extract)
print(result)
处理混淆的JS变量
import re
import requests
from urllib.parse import unquote
def extract_obfuscated_vars(url):
response = requests.get(url)
js_content = response.text
# 提取base64编码的变量
base64_vars = re.findall(r'base64_decode\(["\']([^"\']+)["\']\)', js_content)
decoded_vars = []
for b64 in base64_vars:
try:
import base64
decoded = base64.b64decode(b64).decode('utf-8')
decoded_vars.append(decoded)
except:
continue
# 提取URL编码的变量
url_encoded = re.findall(r'decodeURIComponent\(["\']([^"\']+)["\']\)', js_content)
for url_enc in url_encoded:
decoded = unquote(url_enc)
decoded_vars.append(decoded)
return decoded_vars
综合使用示例
import asyncio
from pyppeteer import launch
async def async_js_extractor(url):
browser = await launch(headless=True)
page = await browser.newPage()
await page.goto(url)
# 等待JS加载完成
await page.waitForSelector('script')
# 提取变量
variables = await page.evaluate('''
() => {
return {
token: window.token,
config: window.config,
userInfo: window.userInfo
}
}
''')
await browser.close()
return variables
# 运行
async def main():
result = await async_js_extractor('https://example.com')
print(result)
# asyncio.run(main())
注意事项
- 反爬虫处理：添加User-Agent、Referer等请求头
- 动态加载：使用Selenium或Pyppeteer处理动态JS
- 错误处理：添加异常捕获，处理变量不存在的情况
- 性能优化：考虑使用连接池、缓存等
选择哪种方法取决于具体的页面结构和JavaScript变量的加载方式。
