整一个脚本：将参与投票者的信息导出到csv - 周边问答

teddyxlandlee

需要在有“查看投票参与人”按钮的投票帖中加一个按钮“导出至CSV”

csv表头：

Type,TotalCredits,Nuggets,Emeralds,DingTies,Nickname,UserGroupName,UID

复制代码

依次为：投票项目、经验总数、金粒、宝石、顶帖数、昵称、等级名称、UID。

其中UID从用户名的那个链接（href）中截取，经验总数、金粒、宝石、昵称、等级名称从MBW的积分查询API中获取，顶帖数用xmdhs的api：

https://auto.xmdhs.com/getforuid?uid=<UID>

复制代码

破损的鞘翅

// ==UserScript==
// @name mcbbs:将参与投票者的信息导出到csv
// @namespace http://tampermonkey.net/
// @version 0.1
// @description https://www.mcbbs.net/thread-1464590-1-1.html
// @author 破损的鞘翅
// @match https://www.mcbbs.net/thread-*
// @match https://www.mcbbs.net/forum.php?mod=viewthread*
// @icon https://www.mcbbs.net/favicon.ico
// @grant none
// ==/UserScript==
(function () {
"use strict";
// Your code here...
class CSV {
headers = [];
datas = [];
width = 0;
constructor(...headers) {
this.headers = headers.join(",");
this.width = headers.length;
}
add(...data) {
while (data.length < this.width) {
data.push("");
}
this.datas.push(data.join(","));
}
saveAs(filename) {
const a = document.createElement("a");
a.href = URL.createObjectURL(new Blob([this.export], { type: "text/plain" }));
a.download = filename + ".csv";
document.body.append(a);
a.click();
a.remove();
}
get export() {
const exports = [this.headers, ...this.datas];
return exports.join("\r\n");
}
}
async function fetchXml(url, init = {}) {
const domParser = new DOMParser();
const response = await fetch(url, init);
const resText = await response.text();
const resXml = domParser.parseFromString(resText, "text/xml");
return domParser.parseFromString(resXml.documentElement.childNodes[0].data, "text/html");
}
async function getUserInfo(uid) {
const res = await fetch(`https://www.mcbbs.net/home.php?mod=space&uid=${uid}`).then((res) => res.text());
const html = new DOMParser().parseFromString(res, "text/html");
try {
return {
uid,
exp: html.querySelector("#psts>ul>li:nth-child(2)").childNodes[1].nodeValue.match(/[0-9]{1,}/)[0],
nuggets: html.querySelector("#psts>ul>li:nth-child(4)").childNodes[1].nodeValue.match(/[0-9]{1,}/)[0],
emeralds: html.querySelector("#psts>ul>li:nth-child(6)").childNodes[1].nodeValue.match(/[0-9]{1,}/)[0],
nickname: html.querySelector("#uhd .mt").innerText.replace(/\n/g, ""),
group: html.querySelector("#pbbs").previousElementSibling.querySelector("li:last-child a").innerText,
};
} catch {
console.error(`获取用户 ${uid} 的信息失败。`);
}
}
async function getDingTie(uid) {
const result = await fetch(`https://auto.xmdhs.com/getforuid?uid=${uid}`).then((res) => res.json());
return result.data?.length || 0;
}
async function getVoteResult(tid, page = 1, optionId = 0) {
const result = []; //单个选项的投票结果
const classifiedResult = {}; //投票结果的集合
const url = new URL(`https://www.mcbbs.net/forum.php?mod=misc&action=viewvote&tid=${tid}&handlekey=viewvote&inajax=1`);
url.searchParams.set("page", page);
if (optionId > 0) {
//optionId 大于0，为url添加参数，获取对应选项的投票结果。不加该参数则是获取第一个选项的结果。
url.searchParams.set("polloptionid", optionId);
}
const html = await fetchXml(url.href);
for (const li of html.querySelector("ul").children) {
//将结果放入数组
const uid = new URL(li.querySelector("a").href).searchParams.get("uid");
result.push(uid);
}
if (html.querySelector(".nxt")) {
//含有该元素，说明还有下一页，获取下一页的结果，放入数组
const nextRes = await getVoteResult(tid, page + 1, optionId);
result.push(...nextRes);
}
if (optionId > 0 || page > 1) {
//optionId 大于 0 且 page 大于 1，说明该调用只获取单个选项的投票结果，返回之前获取的结果
return result;
} else {
//反之，说明该调用是获取全部的投票结果。除了已经获取了的默认的结果，再获取其他选项的结果
const options = html.querySelector("select").children;
for (const option of options) {
if (option.selected) {
classifiedResult[option.innerText] = result;
continue;
}
classifiedResult[option.innerText] = await getVoteResult(tid, 1, option.value);
}
return classifiedResult;
}
}
function getButton() {
const btn = document.createElement("a");
btn.href = "javascript:void();";
btn.addEventListener("click", async () => {
showPrompt(null, null, "开始获取数据，在此期间请勿再次点击本按钮", 5000);
try {
const csv = new CSV("Type", "TotalCredits", "Nuggets", "Emeralds", "DingTies", "Nickname", "UserGroupName", "UID");
const result = await getVoteResult(window.tid);
for (const option in result) {
for (const uid of result[option]) {
console.log("正在获取用户:", uid);
const { exp, nuggets, emeralds, nickname, group } = await getUserInfo(uid);
const dingTie = await getDingTie(uid);
csv.add(option, exp, nuggets, emeralds, dingTie, nickname, group, uid);
}
}
csv.saveAs(document.querySelector("#thread_subject").innerText);
showPrompt(null, null, "数据获取完毕，确认数据完整前请勿关闭网页，若数据有缺损，请打开控制台查看错误信息", 5000);
} catch (err) {
showPrompt(null, null, "数据获取出错，请打开控制台查看错误信息", 5000);
throw err;
}
});
btn.innerText = "将数据导出至CSV";
return btn;
}
if (document.querySelector("#poll>.pinf") && document.querySelector("#poll>.pinf").innerText.search("查看投票参与人") > 1) {
document.querySelector("#poll>.pinf").append(getButton());
}
})();

复制代码

teddyxlandlee

破损的鞘翅发表于 2023-8-17 16:41
要获取的数据比较多，但请求数据的方式没有采用并发式的，而是等前面的数据获取完了再获取下一个。一是懒 ...

跨域我可以手动开，如果可以的话还是用mbw的api吧

破损的鞘翅

teddyxlandlee 发表于 2023-8-17 17:59
跨域我可以手动开，如果可以的话还是用mbw的api吧
因为mbw有缓存，不怕打（确信） ...

// ==UserScript==
// @name mcbbs:将参与投票者的信息导出到csv
// @namespace http://tampermonkey.net/
// @version 0.1
// @description https://www.mcbbs.net/thread-1464590-1-1.html
// @author 破损的鞘翅
// @match https://www.mcbbs.net/thread-*
// @match https://www.mcbbs.net/forum.php?mod=viewthread*
// @icon https://www.mcbbs.net/favicon.ico
// @grant none
// ==/UserScript==
(function () {
"use strict";
// Your code here...
class CSV {
headers = [];
datas = [];
width = 0;
constructor(...headers) {
this.headers = headers.join(",");
this.width = headers.length;
}
add(...data) {
while (data.length < this.width) {
data.push("");
}
this.datas.push(data.join(","));
}
saveAs(filename) {
const a = document.createElement("a");
a.href = URL.createObjectURL(new Blob([this.export], { type: "text/plain" }));
a.download = filename + ".csv";
document.body.append(a);
a.click();
a.remove();
}
get export() {
const exports = [this.headers, ...this.datas];
return exports.join("\r\n");
}
}
async function fetchXml(url, init = {}) {
const domParser = new DOMParser();
const response = await fetch(url, init);
const resText = await response.text();
const resXml = domParser.parseFromString(resText, "text/xml");
return domParser.parseFromString(resXml.documentElement.childNodes[0].data, "text/html");
}
async function getUserInfoByApi(uid) {
const url = `https://mcbbs.wiki/rest.php/mbwutils/v0/credit/${uid}`;
const {
notfound,
nickname = "",
credits = {},
activities: { currentGroupText },
} = await fetch(url).then((res) => res.json());
if (notfound) {
throw false;
}
return {
uid,
exp: credits.credit,
nuggets: credits.nugget,
emeralds: credits.gem,
nickname,
group: currentGroupText,
};
}
async function getDingTie(uid) {
const result = await fetch(`https://auto.xmdhs.com/getforuid?uid=${uid}`).then((res) => res.json());
return result.data?.length || 0;
}
async function getVoteResult(tid, page = 1, optionId = 0) {
const result = []; //单个选项的投票结果
const classifiedResult = {}; //投票结果的集合
const url = new URL(`https://www.mcbbs.net/forum.php?mod=misc&action=viewvote&tid=${tid}&handlekey=viewvote&inajax=1`);
url.searchParams.set("page", page);
if (optionId > 0) {
//optionId 大于0，为url添加参数，获取对应选项的投票结果。不加该参数则是获取第一个选项的结果。
url.searchParams.set("polloptionid", optionId);
}
const html = await fetchXml(url.href);
for (const li of html.querySelector("ul").children) {
//将结果放入数组
const uid = new URL(li.querySelector("a").href).searchParams.get("uid");
result.push(uid);
}
if (html.querySelector(".nxt")) {
//含有该元素，说明还有下一页，获取下一页的结果，放入数组
const nextRes = await getVoteResult(tid, page + 1, optionId);
result.push(...nextRes);
}
if (optionId > 0 || page > 1) {
//optionId 大于 0 且 page 大于 1，说明该调用只获取单个选项的投票结果，返回之前获取的结果
return result;
} else {
//反之，说明该调用是获取全部的投票结果。除了已经获取了的默认的结果，再获取其他选项的结果
const options = html.querySelector("select").children;
for (const option of options) {
if (option.selected) {
classifiedResult[option.innerText] = result;
continue;
}
classifiedResult[option.innerText] = await getVoteResult(tid, 1, option.value);
}
return classifiedResult;
}
}
function getButton() {
const btn = document.createElement("a");
btn.href = "javascript:void();";
btn.addEventListener("click", async () => {
showPrompt(null, null, "开始获取数据，在此期间请勿再次点击本按钮", 5000);
try {
const csv = new CSV("Type", "TotalCredits", "Nuggets", "Emeralds", "DingTies", "Nickname", "UserGroupName", "UID");
const result = await getVoteResult(window.tid);
let taskQueueChunk = [];
for (const option in result) {
for (const uid of result[option]) {
taskQueueChunk.push(
(async () => {
try {
const { exp, nuggets, emeralds, nickname, group } = await getUserInfoByApi(uid);
const dingTie = await getDingTie(uid);
csv.add(option, exp, nuggets, emeralds, dingTie, nickname, group, uid);
} catch {
throw uid;
}
})()
);
if (taskQueueChunk.length >= 5) {
//每5个promise一组
await Promise.allSettled(taskQueueChunk);
taskQueueChunk = [];
}
}
}
csv.saveAs(document.querySelector("#thread_subject").innerText);
showPrompt(null, null, "数据获取完毕，确认数据完整前请勿关闭网页，若数据有缺损，请打开控制台查看错误信息", 5000);
} catch (err) {
showPrompt(null, null, "数据获取出错，请打开控制台查看错误信息", 5000);
throw err;
}
});
btn.innerText = "将数据导出至CSV";
return btn;
}
if (document.querySelector("#poll>.pinf") && document.querySelector("#poll>.pinf").innerText.search("查看投票参与人") > 1) {
document.querySelector("#poll>.pinf").append(getButton());
}
})();

复制代码

现在是5个5个一组，感觉问题不大

另外，解析html确实算是下策，不确定因素比较多
不知道bbs本身有没有api可以用
我永远喜欢前后端分离（

teddyxlandlee

破损的鞘翅发表于 2023-8-17 09:07
现在是5个5个一组，感觉问题不大

另外，解析html确实算是下策，不确定因素比较多

追加：

Uncaught SyntaxError: expected expression, got ')'

破损的鞘翅

teddyxlandlee 发表于 2023-8-18 09:44
Uncaught (in promise) TypeError: html.querySelector(...) is null

你正在爬的投票贴是哪个？调试页里的网络tab那边有没有红色的请求

teddyxlandlee

破损的鞘翅发表于 2023-8-18 12:34
你正在爬的投票贴是哪个？调试页里的网络tab那边有没有红色的请求

我的帖子列表中最新的那个，感谢

破损的鞘翅

teddyxlandlee 发表于 2023-8-18 12:36
我的帖子列表中最新的那个，感谢

// ==UserScript==
// @name mcbbs:将参与投票者的信息导出到csv
// @namespace http://tampermonkey.net/
// @version 0.3
// @description https://www.mcbbs.net/thread-1464590-1-1.html
// @author 破损的鞘翅
// @match https://www.mcbbs.net/thread-*
// @match https://www.mcbbs.net/forum.php?mod=viewthread*
// @icon https://www.mcbbs.net/favicon.ico
// @grant none
// ==/UserScript==
(function () {
"use strict";
// Your code here...
class CSV {
headers = [];
datas = [];
width = 0;
constructor(...headers) {
this.headers = headers.join(",");
this.width = headers.length;
}
add(...data) {
while (data.length < this.width) {
data.push("");
}
this.datas.push(data.join(","));
}
saveAs(filename) {
const a = document.createElement("a");
a.href = URL.createObjectURL(new Blob([this.export], { type: "text/plain" }));
a.download = filename + ".csv";
document.body.append(a);
a.click();
a.remove();
}
get export() {
const exports = [this.headers, ...this.datas];
return exports.join("\r\n");
}
}
async function fetchXml(url, init = {}) {
const domParser = new DOMParser();
const response = await fetch(url, init);
const resText = (await response.text()) || "";
const resXml = domParser.parseFromString(resText, "text/xml");
return {
html: domParser.parseFromString(resXml.documentElement.childNodes[0].data, "text/html"),
raw: resText,
};
}
async function getUserInfoByApi(uid) {
const url = `https://mcbbs.wiki/rest.php/mbwutils/v0/credit/${uid}`;
const {
notfound,
nickname = "",
credits = {},
activities: { currentGroupText },
} = await fetch(url)
.then((res) => res.json())
.catch(() => {});
if (notfound) {
throw false;
}
return {
uid,
exp: credits.credit,
nuggets: credits.nugget,
emeralds: credits.gem,
nickname,
group: currentGroupText,
};
}
async function getDingTie(uid) {
console.log(uid);
const result = await fetch(`https://auto.xmdhs.com/getforuid?uid=${uid}`).then((res) => res.json());
return result.data?.length || 0;
}
async function getVoteResult(tid, page = 1, optionId = 0) {
const result = []; //单个选项的投票结果
const classifiedResult = {}; //投票结果的集合
const url = new URL(`https://www.mcbbs.net/forum.php?mod=misc&action=viewvote&tid=${tid}&handlekey=viewvote&inajax=1`);
url.searchParams.set("page", page);
if (optionId > 0) {
//optionId 大于0，为url添加参数，获取对应选项的投票结果。不加该参数则是获取第一个选项的结果。
url.searchParams.set("polloptionid", optionId);
}
let { html, raw } = await fetchXml(url.href);
while (html.querySelector("ul") == null) {
//没获取到数据，1s后重试
console.warn(`获取 ${url.href} 的数据失败:`, { raw });
await new Promise((resolve) => setTimeout(resolve, 1000));
({ html, raw } = await fetchXml(url.href));
}
for (const li of html.querySelector("ul").children) {
//将结果放入数组
const uid = new URL(li.querySelector("a").href).searchParams.get("uid");
result.push(uid);
}
if (html.querySelector(".nxt")) {
//含有该元素，说明还有下一页，获取下一页的结果，放入数组
const nextRes = await getVoteResult(tid, page + 1, optionId);
result.push(...nextRes);
}
if (optionId > 0 || page > 1) {
//optionId 大于 0 且 page 大于 1，说明该调用只获取单个选项的投票结果，返回之前获取的结果
return result;
} else {
//反之，说明该调用是获取全部的投票结果。除了已经获取了的默认的结果，再获取其他选项的结果
const options = html.querySelector("select").children;
for (const option of options) {
if (option.selected) {
classifiedResult[option.innerText] = result;
continue;
}
classifiedResult[option.innerText] = await getVoteResult(tid, 1, option.value);
}
return classifiedResult;
}
}
function getButton() {
const btn = document.createElement("a");
btn.href = "javascript:;";
btn.addEventListener("click", async () => {
showPrompt(null, null, "开始获取数据，在此期间请勿再次点击本按钮", 5000);
try {
const csv = new CSV("Type", "TotalCredits", "Nuggets", "Emeralds", "DingTies", "Nickname", "UserGroupName", "UID");
const result = await getVoteResult(window.tid);
let taskQueueChunk = [];
for (const option in result) {
for (const uid of result[option]) {
taskQueueChunk.push(
(async () => {
try {
const { exp, nuggets, emeralds, nickname, group } = await getUserInfoByApi(uid);
const dingTie = await getDingTie(uid);
csv.add(option, exp, nuggets, emeralds, dingTie, nickname, group, uid);
} catch {
throw uid;
}
})()
);
if (taskQueueChunk.length >= 5) {
//每5个promise一组
await Promise.allSettled(taskQueueChunk);
taskQueueChunk = [];
}
}
}
//等待没凑满5个的队列兑现
await Promise.allSettled(taskQueueChunk);
csv.saveAs(document.querySelector("#thread_subject").innerText);
showPrompt(null, null, "数据获取完毕，确认数据完整前请勿关闭网页，若数据有缺损，请打开控制台查看错误信息", 5000);
} catch (err) {
showPrompt(null, null, "数据获取出错，请打开控制台查看错误信息", 5000);
throw err;
}
});
btn.innerText = "将数据导出至CSV";
return btn;
}
if (document.querySelector("#poll>.pinf") && document.querySelector("#poll>.pinf").innerText.search("查看投票参与人") > 1) {
document.querySelector("#poll>.pinf").append(getButton());
}
})();

复制代码

加入了一个没获取到投票结果时就等1s后重试的功能
我猜测是你那边由于奇怪的网络原因，导致获取投票结果的请求没能返回正常的数据。现在如果数据有异常会打印到控制台

teddyxlandlee

破损的鞘翅发表于 2023-8-18 13:23
加入了一个没获取到投票结果时就等1s后重试的功能
我猜测是你那边由于奇怪的网络原因，导致获取投票结果 ...

已拦截跨源请求：同源策略禁止读取位于 https://mcbbs.wiki/rest.php/mbwutils/v0/credit/******* 的远程资源。（原因：CORS 头缺少 'Access-Control-Allow-Origin'）。状态码：200。

既然跨源被浏览器拦截了，那隔壁的SPXX为什么可以跨源获取漏洞翻译的文本呢？

破损的鞘翅

teddyxlandlee 发表于 2023-8-18 13:52
既然跨源被浏览器拦截了，那隔壁的SPXX为什么可以跨源获取漏洞翻译的文本呢？

↑这个用@connect和@grant ...

那个不是单靠@connect就能搞定的，还要用@grant请求GM_xmlhttpRequest权限，然后用GM_xmlhttpRequest这个方法去发出请求，只有用这个方法，油猴才知道你是在向谁发出请求，然后检查目标域名有没有被connect,有的话，油猴以扩展程序的身份去发起请求，再把数据还给脚本。油猴相当于代理。但是这种请求，控制台里是看不见的。单纯connect，脚本内直接用fetch，实际上还是会被跨域策略阻止。
window没了，因为用了@grant，这时候用unsafeWindow代替window就行

teddyxlandlee

破损的鞘翅发表于 2023-8-18 14:33
那个不是单靠@connect就能搞定的，还要用@grant请求GM_xmlhttpRequest权限，然后用GM_xmlhttpRequest这个 ...

那么为什么xmdhs的提升卡记录，直接用的fetch却可以访问，mcbbs.wiki的就会拦截？

下一页最后一页