可以修改为你的下载地址
const { HttpsProxyAgent } = require('https-proxy-agent');
const fetch = require('node-fetch');
const fs = require('fs');
function getFileNameFromContentDisposition(contentDispositionHeader) {
// 匹配 filename="..." 部分的正则表达式
const filenameRegex = /filename="([^"]+)"/;
const match = contentDispositionHeader.match(filenameRegex);
if (match && match.length > 1) {
const filenameWithQuotes = match[1];
// 去掉双引号
const filename = filenameWithQuotes.replace(/\"/g, '');
return filename;
} else {
return null; // 没有匹配到文件名
}
}
async function fetchpixeldrainPage(url) {
return new Promise(async (resolve, reject) => { // 加上async关键字
try {
// 根据需要选择使用 HttpProxyAgent 或 HttpsProxyAgent
const proxyAgent = new HttpsProxyAgent('http://127.0.0.1:7890');
const pattern = /https:\/\/pixeldrain\.com\/u\/([^/\s]+)/;
const match = url.match(pattern);
if (match) {
const extractedString = match[1];
try {
// 使用axios获取URL的内容
const response = await fetch(`https://pixeldrain.com/api/file/${extractedString}?download`, {
agent: proxyAgent,
headers: {
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
'Accept-Encoding': 'gzip, deflate, br',
'Accept-Language': 'zh-CN,zh;q=0.9',
'Connection': 'keep-alive',
'Cookie': 'lng=eng; _gid=GA1.2.277384685.1704939305; _ga=GA1.2.849566254.1704847242; _ga_LT9YQX0N49=GS1.1.1705046293.8.1.1705046298.0.0.0',
'Host': 'pixeldrain.com',
'Sec-Ch-Ua': '"Not_A Brand";v="8", "Chromium";v="120", "Google Chrome";v="120"',
'Sec-Ch-Ua-Mobile': '?0',
'Sec-Ch-Ua-Platform': '"Windows"',
'Sec-Fetch-Dest': 'document',
'Sec-Fetch-Mode': 'navigate',
'Sec-Fetch-Site': 'none',
'Sec-Fetch-User': '?1',
'Upgrade-Insecure-Requests': '1',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
},
});
const contentDispositionHeader = response.headers.get('Content-Disposition');
const fileName = getFileNameFromContentDisposition(contentDispositionHeader);
if (!fileName.endsWith('.txt')) {
console.log(`${fileName} 不是txt文件, ${extractedString} `);
resolve()
} else {
// 创建可写流
const fileStream = fs.createWriteStream('F:\\1\\' + fileName);
// 获取响应头中的 Content-Length
const contentLength = response.headers.get('Content-Length');
const fileSizeInMB = contentLength / (1024 * 1024);
let downloadedBytes = 0;
// 监听数据流事件
response.body.on('data', (chunk) => {
downloadedBytes += chunk.length;
const percentage = Math.round((downloadedBytes / contentLength) * 100);
console.log(`${fileName} 下载进度: ${percentage}% ${fileSizeInMB.toFixed(2)} 'MB'`);
});
// 将响应数据流写入文件
response.body.pipe(fileStream);
// 监听文件写入完成事件
fileStream.on('finish', () => {
console.log('内容已保存到文件:', fileName);
resolve(); // 下载完成后解析 Promise
});
console.log('内容已保存到文件:', fileName);
}
} catch (error) {
console.error('Error fetching or saving URL:', error.message, url);
reject(error); // 发生错误时拒绝 Promise
}
} else {
console.log('URL不匹配模式。');
reject(error); // 发生错误时拒绝 Promise
}
} catch (error) {
console.error('Error fetching or processing the page:', error);
reject(error); // 发生错误时拒绝 Promise
}
});
}
async function main(startIndex, txtlist, batchSize, check) {
const promises = [];
for (let i = startIndex; i < startIndex + batchSize && i <= txtlist.length; ++i) {
promises.push(fetchpixeldrainPage(`${txtlist[i]}`));
}
await Promise.all(promises);
console.log(`Links extracted and saved to for batch starting from ${startIndex}`);
if (startIndex + batchSize <= txtlist.length) {
// 开启下一批异步任务
await main(startIndex + batchSize, txtlist, batchSize, check);
}
}
async function readTxt(path) {
// 使用fs.readFileSync同步读取文件内容
const fileContent = fs.readFileSync(path, 'utf8');
// 将文件内容按行分割,并存储到数组中
const lines = fileContent.split('\n');
return lines
}
async function startMain() {
const batchSize = 1;
let check = "pixel",txtlist =[]
txtlist = await readTxt('search_results_pixeldrain.txt')
// 启动第一批异步任务
await main(0, txtlist, batchSize, check);
}
startMain();