TestController.cs 90 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196
  1. using molilian.core;
  2. using dodohold.core;
  3. using Dapper;
  4. using Microsoft.AspNetCore.Mvc;
  5. using Org.BouncyCastle.Ocsp;
  6. using System.Collections.Concurrent;
  7. using System.Diagnostics;
  8. using System.Text.Json;
  9. using System.Runtime.InteropServices;
  10. using System.Net;
  11. using System.Threading;
  12. using System.Data;
  13. using Microsoft.AspNetCore.Mvc.RazorPages;
  14. using TencentCloud.Soe.V20180724.Models;
  15. using static dodohold.core.ZTOExpress.CreateOrderArgs;
  16. using static Spire.Xls.Core.Spreadsheet.HTMLOptions;
  17. using System.Net.Http.Json;
  18. using System.Text.RegularExpressions;
  19. using System.Text;
  20. using TencentCloud.Oceanus.V20190422.Models;
  21. using YunhuiKit;
  22. using TencentCloud.Cdn.V20180606.Models;
  23. namespace molilian.api.Controllers
  24. {
  25. [ApiController]
  26. [Route("[controller]/[action]")]
  27. public class TestController : ControllerBase
  28. {
  29. private const string JdStressDeeplinkContent = "openapp.jdmobile://virtual?params=%7B%22category%22%3A%22jump%22%2C%22des%22%3A%22productDetail%22%2C%22skuId%22%3A%2210139003162781%22%2C%22sourceType%22%3A%22Item%22%2C%22sourceValue%22%3A%22view-ware%22%7D";
  30. private const int DeeplinkReportStatsExpireSeconds = 90 * 86400;
  31. private const int DeeplinkReportRepairMaxDays = 366;
  32. protected IHttpContextAccessor _accessor;
  33. public TestController(IHttpContextAccessor accessor)
  34. {
  35. _accessor = accessor;
  36. }
  37. [HttpGet]
  38. public async Task<ActionResult> jd_unsafe_parse_stress([FromQuery] int t = 5, [FromQuery] int time = 60, [FromQuery] string ip = "127.0.0.1", [FromQuery] string oaid = "", [FromQuery] bool wait = false)
  39. {
  40. t = t <= 0 ? 5 : t;
  41. time = time <= 0 ? 60 : time;
  42. string runId = DateTime.Now.ToString("yyyyMMdd_HHmmss_fff");
  43. string logName = $"run_{runId}_t{t}_time{time}";
  44. string startLogStatus = await SaveJdStressStartLogAsync(runId, logName, t, time, ip, oaid, wait);
  45. if (wait)
  46. {
  47. var result = await RunJdStressAsync(runId, logName, t, time, ip, oaid, startLogStatus, CancellationToken.None);
  48. return new APIResult(result);
  49. }
  50. _ = Task.Run(async () =>
  51. {
  52. try
  53. {
  54. await RunJdStressAsync(runId, logName, t, time, ip, oaid, startLogStatus, CancellationToken.None);
  55. }
  56. catch (Exception ex)
  57. {
  58. _ = new LoggerLibrary("jd_api_stress", $"{logName}_runner_error")
  59. .Info(ex.Message, ex.StackTrace)
  60. .SaveAsync();
  61. }
  62. });
  63. return new APIResult(new
  64. {
  65. success = true,
  66. message = "后台压测已启动",
  67. runId,
  68. requestPerSecond = t,
  69. timeSeconds = time,
  70. expectedRequests = (long)t * time,
  71. wait,
  72. log = new
  73. {
  74. type = "LoggerLibrary",
  75. dir = "jd_api_stress",
  76. name = logName,
  77. status = startLogStatus
  78. }
  79. });
  80. }
  81. [HttpGet]
  82. public async Task<ActionResult> ecs_list_test()
  83. {
  84. var list = AliyunPoolCore.EcsList();
  85. return new APIResult(new
  86. {
  87. success = true,
  88. msg = "ok",
  89. list
  90. });
  91. }
  92. [HttpGet]
  93. public async Task<ActionResult> testReconnectionRedis()
  94. {
  95. string cacheKey = "test";
  96. RedisKit.SetAsync(cacheKey, 1, 3600);
  97. string val = await RedisKit.GetAsync<string>(cacheKey);
  98. return new APIResult(new { success = "ok", val });
  99. }
  100. [HttpGet]
  101. public async Task<ActionResult> backfill_track_parse_metrics([FromQuery] string reportDate = "")
  102. {
  103. DateTime targetDate = DateTime.Now.Date;
  104. if (!string.IsNullOrWhiteSpace(reportDate) && !DateTime.TryParse(reportDate, out targetDate))
  105. {
  106. return new APIResult(new
  107. {
  108. success = false,
  109. message = "reportDate格式错误,请使用 yyyy-MM-dd"
  110. });
  111. }
  112. var result = await TracksCore.BackfillParseMetricCountersAsync(targetDate);
  113. return new APIResult(new
  114. {
  115. success = true,
  116. message = "ok",
  117. data = result
  118. });
  119. }
  120. [HttpGet]
  121. public async Task<ActionResult> ChangePublicIpByName(string nodeName)
  122. {
  123. var proxy_node = new DBContext.Table("proxy_nodes").Get<dynamic>("nodeName=@nodeName", new { nodeName });
  124. if (proxy_node == null) return new APIResult(new { success = false, msg = "没有匹配的 proxy_nodes 记录" });
  125. int aliyun_id = proxy_node.aliyun_id;
  126. string proxy_server = proxy_node.server;
  127. var account = new DBContext.Table("aliyun_pool").Get<AliyunPoolDTO>(aliyun_id);
  128. if (account == null) return new APIResult(new { success = false, msg = "没有匹配的 aliyun_pool 记录" });
  129. var uri = new Uri(proxy_server);
  130. string privateIp = uri.Host;
  131. AliyunCore core = new AliyunCore(account);
  132. var success = await core.ChangePublicIpAsync(privateIp);
  133. return new APIResult(new { success = "ok" });
  134. }
  135. [HttpGet]
  136. public async Task<ActionResult> ChangePublicIp(int id)
  137. {
  138. id = id >= 20000 ? id - 20000 : id;
  139. var proxy_node = new DBContext.Table("proxy_nodes").Get<dynamic>(id);
  140. if (proxy_node == null) return new APIResult(new { success = false, msg = "没有匹配的 proxy_nodes 记录" });
  141. int aliyun_id = proxy_node.aliyun_id;
  142. string proxy_server = proxy_node.server;
  143. var account = new DBContext.Table("aliyun_pool").Get<AliyunPoolDTO>(aliyun_id);
  144. if (account == null) return new APIResult(new { success = false, msg = "没有匹配的 aliyun_pool 记录" });
  145. var uri = new Uri(proxy_server);
  146. string privateIp = uri.Host;
  147. AliyunCore core = new AliyunCore(account);
  148. var success = await core.ChangePublicIpAsync(privateIp);
  149. return new APIResult(new { success = "ok" });
  150. }
  151. [HttpGet]
  152. //第一部 先加一个网卡,
  153. public async Task<ActionResult> CreateNetworkInterface(int id, string ecsid)
  154. {
  155. AliyunCore core = new AliyunCore(id);
  156. //创建弹性网卡并绑定公网IP
  157. var success = core.EcsCreateNetworkInterface("cn-beijing", ecsid);
  158. return new APIResult(new { success });
  159. }
  160. [HttpGet]
  161. //第二部 给网卡绑定很多个辅助ip
  162. public async Task<ActionResult> EcsAssignPrivateIpAddresses(int id, string ecsid, int count, bool isSecondary = true)
  163. {
  164. AliyunCore core = new AliyunCore(id);
  165. //创建弹性网卡并绑定公网IP
  166. var success = core.EcsAssignPrivateIpAddresses("cn-beijing", ecsid, count, !isSecondary);
  167. return new APIResult(new { success });
  168. }
  169. [HttpGet]
  170. public async Task<ActionResult> QueryAccountBalance()
  171. {
  172. AliyunPlus plus = new AliyunPlus("LTAI5tQbkTjtULQcrWGaw2VJ", "UIkkolVVEddooOKOIUByCqymZkK6ZA");
  173. var response = plus.QueryAccountBalance();
  174. var balance = response.Body.Data.AvailableAmount;
  175. return new APIResult(new { response });
  176. }
  177. //[HttpGet]
  178. //public async Task<ActionResult> test(string table = "tk_parse_logs_shop", int count = 100)
  179. //{
  180. // //table = "tk_parse_logs_shop";
  181. // //table = "tk_parse_logs_live";
  182. // //table = "tk_parse_logs_video";
  183. // string filter = "success=0 AND message='放弃转链' AND reason='非标准链接'";
  184. // for (int i = 0; i < 100; i++)
  185. // {
  186. // try
  187. // {
  188. // var list = new DBContext.Table(table)
  189. // .Where(filter, new { })
  190. // .Limit(count).Select<TkDataDTO>();
  191. // if (!list.Any())
  192. // {
  193. // return new APIResult(new
  194. // {
  195. // success = false,
  196. // message = "所有任务完成"
  197. // });
  198. // }
  199. // foreach (var item in list)
  200. // {
  201. // TkPoolDTO? account = TkPoolCore.GetOne(TkPoolCore.TkAction.parse);
  202. // if (account == null)
  203. // {
  204. // return new APIResult(new
  205. // {
  206. // success = false,
  207. // message = "没有工作账号"
  208. // });
  209. // }
  210. // var result = item.Convert2Json().Convert2Object<TkDataDTO>();
  211. // result.accountId = account.id;
  212. // result.accountName = account.company;
  213. // var alimama = new AlimamaPlus(account);
  214. // int timeout = alimama._config.rt_max;
  215. // using var cts = new CancellationTokenSource();
  216. // cts.CancelAfter(timeout);
  217. // result = await alimama.UnionParseAsync(item.content, result);
  218. // string reason = "非标准链接".Equals(result.reason) ? "非标准链接ok" : result.reason;
  219. // new DBContext.Table(table)
  220. // .Add("linkType", (int)result.link_type)
  221. // .Add("rawContent", result.rawContent)
  222. // .Add("success", result.success)
  223. // .Add("message", result.message)
  224. // .Add("reason", reason)
  225. // .Add("content", result.content)
  226. // .Add("itemId", result.itemId)
  227. // .Add("itemName", result.itemName)
  228. // .Add("pic", result.pic)
  229. // .Add("couponAmount", result.couponAmount)
  230. // .Add("promotionPrice", result.promotionPrice)
  231. // .Add("taoToken", result.taoToken)
  232. // .Add("shortLinkurl", result.shortLinkurl)
  233. // .Add("deeplink_url", result.deeplink_url)
  234. // .Add("num_iid", result.num_iid)
  235. // .Add("elapsedTime", result.elapsedTime)
  236. // .Add("elapsedTime2", result.elapsedTime2)
  237. // .Add("elapsedTime3", result.elapsedTime3)
  238. // .Add("subCode", result.subCode)
  239. // .Where("id=@id", new { item.id })
  240. // .Update();
  241. // }
  242. // }
  243. // catch
  244. // {
  245. // }
  246. // }
  247. // return new APIResult(new
  248. // {
  249. // success = true,
  250. // message = "ok"
  251. // });
  252. //}
  253. [HttpGet]
  254. public async Task<ActionResult> redu_douyin(string command)
  255. {
  256. ReduPlus plus = new ReduPlus("", "", "");
  257. var result = await plus.DouyinParse(command);
  258. return Content(result.Convert2Json());
  259. }
  260. [HttpGet]
  261. public async Task<ActionResult> redu_kuaishou(string command)
  262. {
  263. ReduPlus plus = new ReduPlus("", "", "");
  264. var result = await plus.KuaishouParse(command);
  265. return Content(result.Convert2Json());
  266. }
  267. [HttpPost]
  268. public async Task<ActionResult> test2([FromForm] string jsonContent, [FromForm] string testText)
  269. {
  270. // 用于存储输出结果的StringBuilder
  271. StringBuilder outputBuilder = new StringBuilder();
  272. // 解析JSON数据
  273. JsonDocument jsonDoc = JsonDocument.Parse(jsonContent);
  274. // 获取根元素
  275. JsonElement root = jsonDoc.RootElement;
  276. // 遍历规则
  277. foreach (JsonElement ruleSet in root.EnumerateArray())
  278. {
  279. string platformType = ruleSet.Read("platformType", string.Empty);
  280. string supplier = ruleSet.Read("supplier", string.Empty);
  281. outputBuilder.AppendLine($"平台类型: {platformType}, 供应商: {supplier}");
  282. JsonElement pwdRules = ruleSet.GetProperty("pwdRules");
  283. int idx = 0;
  284. foreach (JsonElement patternElement in pwdRules.EnumerateArray())
  285. {
  286. string pattern = patternElement.GetString();
  287. try
  288. {
  289. // 使用Regex类来编译正则表达式
  290. Regex compiledPattern = new Regex(pattern);
  291. if (compiledPattern.IsMatch(testText))
  292. {
  293. outputBuilder.AppendLine($"规则 {idx + 1}: 匹配\t{pattern}");
  294. }
  295. else
  296. {
  297. outputBuilder.AppendLine($"规则 {idx + 1}: 不匹配");
  298. }
  299. }
  300. catch (Exception e)
  301. {
  302. outputBuilder.AppendLine($"规则 {idx + 1}: 正则表达式错误 - {e.Message}\t{pattern}");
  303. }
  304. idx++;
  305. }
  306. }
  307. outputBuilder.AppendLine("测试完成");
  308. return Content(outputBuilder.ToString());
  309. }
  310. [HttpGet]
  311. public async Task<ActionResult> ip(string ip)
  312. {
  313. string result = IP2RegionPlus.Search(ip);
  314. return new APIResult(new
  315. {
  316. success = true,
  317. message = result
  318. });
  319. }
  320. [HttpPost]
  321. public async Task<ActionResult> testreg([FromBody] JsonElement form)
  322. {
  323. var content = form.Read("s", string.Empty);
  324. string shortLinkurl = AlimamaPlus.GetTaobaoLink(content);
  325. bool is_tao_token = AlimamaPlus.MatchRegexes(content, []);
  326. bool is_other = AlimamaPlus.MatchOtherInfo(content, []);
  327. return new APIResult(new
  328. {
  329. shortLinkurl,
  330. is_tao_token,
  331. is_other
  332. });
  333. }
  334. //[HttpGet]
  335. //public async Task<ActionResult> comparison_tk([FromQuery] int count = 100)
  336. //{
  337. // string cacheKey = ":lock_key:comparison_tk_logs";
  338. // int last_id = RedisHelper.Get<int>(cacheKey);
  339. // string filter = "id>@last_id";
  340. // var result = new DBContext.Table("comparison_tk_logs")
  341. // .Where(filter, new { last_id })
  342. // .Page(count, 1)
  343. // .Order("ID")
  344. // .PageList<TkDataDTO>(false);
  345. // var ip = "127.0.0.1";
  346. // var oaid = "test-comparison_tk_logs";
  347. // foreach (var item in result.List)
  348. // {
  349. // await UnionParseCore.TaobaoParseAsync(item.rawContent, ip, oaid);
  350. // RedisHelper.Set(cacheKey, item.id, 10 * 86400);
  351. // }
  352. // if (result.Count < count)
  353. // {
  354. // cacheKey = ":lock_key:start_comparison_tk";
  355. // RedisHelper.Set(cacheKey, 1, 600);
  356. // }
  357. // return new APIResult(new
  358. // {
  359. // success = true,
  360. // message = "ok"
  361. // });
  362. //}
  363. //[HttpGet]
  364. //public async Task<ActionResult> comparison_tk_raw([FromQuery] int count = 100)
  365. //{
  366. // string cacheKey = ":lock_key:comparison_tk_logs_raw";
  367. // int last_id = RedisHelper.Get<int>(cacheKey);
  368. // string filter = "id>@last_id";
  369. // var result = new DBContext.Table("comparison_tk_logs")
  370. // .Where(filter, new { last_id })
  371. // .Page(count, 1)
  372. // .Order("ID")
  373. // .PageList<TkDataDTO>(false);
  374. // var ip = "127.0.0.1";
  375. // var oaid = "test-comparison_tk_logs_raw";
  376. // foreach (var item in result.List)
  377. // {
  378. // await UnionParseCore.TaobaoParseAsync(item.rawContent, ip, oaid);
  379. // RedisHelper.Set(cacheKey, item.id, 10 * 86400);
  380. // }
  381. // if (result.Count < count)
  382. // {
  383. // cacheKey = ":lock_key:start_comparison_tk";
  384. // RedisHelper.Set(cacheKey, 1, 600);
  385. // }
  386. // return new APIResult(new
  387. // {
  388. // success = true,
  389. // message = "ok"
  390. // });
  391. //}
  392. [HttpGet]
  393. public async Task<ActionResult> testTask()
  394. {
  395. string url = "https://www.taobao.com";
  396. TkDataDTO result = new TkDataDTO();
  397. result.message = "init";
  398. result.success = true;
  399. int timeout = 1000;
  400. using var cts = new CancellationTokenSource();
  401. cts.CancelAfter(timeout);
  402. var requestTask = Task.Run(() => AlimamaPlus.testTaskAsync(result, cts.Token), cts.Token);
  403. var delayTask = Task.Delay(timeout, cts.Token);
  404. var completedTask = await Task.WhenAny(requestTask, delayTask);
  405. if (completedTask == requestTask)
  406. {
  407. result = await requestTask;
  408. }
  409. else
  410. {
  411. cts.Cancel();
  412. result.success = false;
  413. result.message = "放弃转链";
  414. result.reason = "请求超时";
  415. result.itemName = "点击打开淘宝APP";
  416. }
  417. return new APIResult(new
  418. {
  419. success = true,
  420. message = "ok",
  421. result,
  422. });
  423. }
  424. [HttpGet]
  425. public async Task<ActionResult> testThread()
  426. {
  427. // 查看默认的最小和最大线程数
  428. ThreadPool.GetMinThreads(out int defaultMinWorker, out int defaultMinIOC);
  429. ThreadPool.GetMaxThreads(out int defaultMaxWorker, out int defaultMaxIOC);
  430. string tmp = $"Default Min worker threads: {defaultMinWorker}, Min I/O completion threads: {defaultMinIOC}";
  431. string tmp2 = $"Default Max worker threads: {defaultMaxWorker}, Max I/O completion threads: {defaultMaxIOC}";
  432. // 获取当前线程池中可用的工作线程数和 I/O 完成端口线程数
  433. ThreadPool.GetAvailableThreads(out int availableWorkerThreads, out int availableIOCompletionThreads);
  434. string tmp3 = $"Current available worker threads: {availableWorkerThreads}, available I/O completion threads: {availableIOCompletionThreads}";
  435. return new APIResult(new
  436. {
  437. tmp,
  438. tmp2,
  439. tmp3,
  440. });
  441. }
  442. [HttpPost]
  443. public async Task<ActionResult> test1([FromBody] JsonElement form)
  444. {
  445. NotifyCore.Notify(new NifyMessage
  446. {
  447. message = $"【淘宝联盟:test】cookie 掉线",
  448. priority = NifyMessagePriority.high,
  449. tags = ["red_circle"]
  450. });
  451. return new APIResult(new
  452. {
  453. success = true,
  454. message = "ok"
  455. });
  456. }
  457. [HttpGet]
  458. public async Task<ActionResult> RepairDeeplinkReportStats(
  459. DateTime startDate = default,
  460. DateTime endDate = default,
  461. bool dryRun = true,
  462. bool useLegacyRedis = false,
  463. bool allowPartialSources = false,
  464. bool allowCurrentDate = false,
  465. string fallbackEndpoint = "",
  466. int commandTimeoutSeconds = 600)
  467. {
  468. var yesterday = DateTime.Now.Date.AddDays(-1);
  469. if (startDate == default) startDate = yesterday;
  470. if (endDate == default) endDate = startDate;
  471. startDate = startDate.Date;
  472. endDate = endDate.Date;
  473. commandTimeoutSeconds = Math.Clamp(commandTimeoutSeconds, 30, 3600);
  474. if (endDate < startDate)
  475. {
  476. return new APIResult(new { success = false, message = "endDate 不能早于 startDate" });
  477. }
  478. int requestedDays = (endDate - startDate).Days + 1;
  479. if (requestedDays > DeeplinkReportRepairMaxDays)
  480. {
  481. return new APIResult(new
  482. {
  483. success = false,
  484. message = $"单次最多重建 {DeeplinkReportRepairMaxDays} 天数据"
  485. });
  486. }
  487. if (!allowCurrentDate && endDate >= DateTime.Now.Date)
  488. {
  489. return new APIResult(new
  490. {
  491. success = false,
  492. message = "默认禁止重建当天数据,避免覆盖实时计数;如确需执行请传 allowCurrentDate=true"
  493. });
  494. }
  495. var endpoints = (EndPointCore.List(true) ?? Enumerable.Empty<EndPointDTO>())
  496. .Where(node => node.status && node.is_public_api)
  497. .Where(node => !string.IsNullOrWhiteSpace(EndPointCore.GetRedisServer(node)))
  498. .GroupBy(node => node.name, StringComparer.OrdinalIgnoreCase)
  499. .Select(group => group.First())
  500. .OrderBy(node => node.name, StringComparer.OrdinalIgnoreCase)
  501. .ToList();
  502. if (endpoints.Count == 0)
  503. {
  504. return new APIResult(new { success = false, message = "没有可用的公共 API Redis 节点" });
  505. }
  506. EndPointDTO fallbackNode;
  507. if (!string.IsNullOrWhiteSpace(fallbackEndpoint))
  508. {
  509. fallbackNode = endpoints.FirstOrDefault(node => node.name.Equals(
  510. fallbackEndpoint.Trim(),
  511. StringComparison.OrdinalIgnoreCase
  512. ))!;
  513. if (fallbackNode == null)
  514. {
  515. return new APIResult(new
  516. {
  517. success = false,
  518. message = $"fallbackEndpoint={fallbackEndpoint} 不在可用公共节点中",
  519. availableEndpoints = endpoints.Select(node => node.name).ToList()
  520. });
  521. }
  522. }
  523. else
  524. {
  525. fallbackNode = endpoints.FirstOrDefault(node => node.name.Equals(
  526. EndPointCore.CurrentEndPoint,
  527. StringComparison.OrdinalIgnoreCase
  528. )) ?? endpoints[0];
  529. }
  530. var endpointNames = endpoints.ToDictionary(
  531. node => node.name,
  532. node => node.name,
  533. StringComparer.OrdinalIgnoreCase
  534. );
  535. var knownChannelNames = GetKnownDeeplinkReportChannelNames();
  536. var repairDates = new List<DeeplinkReportRepairDate>();
  537. var warnings = new List<object>();
  538. var errors = new List<object>();
  539. if (useLegacyRedis)
  540. {
  541. return await RepairDeeplinkReportStatsFromLegacyRedisAsync(
  542. endpoints,
  543. knownChannelNames,
  544. startDate,
  545. endDate,
  546. dryRun
  547. );
  548. }
  549. var parseAdminEndpoint = EndPointCore.GetParseAdmin();
  550. bool usesConfiguredParseDatabase = parseAdminEndpoint != null &&
  551. !string.IsNullOrWhiteSpace(parseAdminEndpoint.db_server);
  552. using var connection = usesConfiguredParseDatabase
  553. ? EndPointCore.GetDbConnection(parseAdminEndpoint!.db_server)
  554. : DBContext.GetOpenConnection();
  555. if (connection.State != ConnectionState.Open) connection.Open();
  556. foreach (var reportDate in EachDay(startDate, endDate))
  557. {
  558. string dateKey = reportDate.ToString("yyyyMMdd");
  559. string toolTableName = $"tool_parse_logs_{dateKey}";
  560. string deeplinkTableName = $"deeplink_parse_logs_{dateKey}";
  561. bool toolTableExists = TableExists(connection, toolTableName, commandTimeoutSeconds);
  562. bool deeplinkTableExists = TableExists(connection, deeplinkTableName, commandTimeoutSeconds);
  563. if (!toolTableExists && !deeplinkTableExists)
  564. {
  565. warnings.Add(new
  566. {
  567. date = reportDate.ToString("yyyy-MM-dd"),
  568. message = "两个来源日表都不存在,已跳过",
  569. toolTableName,
  570. deeplinkTableName
  571. });
  572. continue;
  573. }
  574. if (!allowPartialSources && (!toolTableExists || !deeplinkTableExists))
  575. {
  576. warnings.Add(new
  577. {
  578. date = reportDate.ToString("yyyy-MM-dd"),
  579. message = "来源日表不完整,已跳过;确认缺失表确实无数据后可传 allowPartialSources=true",
  580. toolTableName,
  581. toolTableExists,
  582. deeplinkTableName,
  583. deeplinkTableExists
  584. });
  585. continue;
  586. }
  587. try
  588. {
  589. var sourceCounts = new List<DeeplinkReportRepairSourceCount>();
  590. if (toolTableExists)
  591. {
  592. var toolCounts = SqlMapper.Query<DeeplinkReportToolSourceRow>(
  593. connection,
  594. $@"
  595. SELECT
  596. COALESCE(end_point, '') end_point,
  597. channel,
  598. COUNT(*) total_count,
  599. CAST(COALESCE(SUM(CASE WHEN success = 1 THEN 1 ELSE 0 END), 0) AS SIGNED) success_count
  600. FROM {toolTableName}
  601. GROUP BY COALESCE(end_point, ''), channel",
  602. commandTimeout: commandTimeoutSeconds
  603. );
  604. sourceCounts.AddRange(toolCounts.Select(item => new DeeplinkReportRepairSourceCount
  605. {
  606. source = toolTableName,
  607. source_endpoint = item.end_point?.Trim() ?? string.Empty,
  608. channel_name = GetDeeplinkReportChannelName(item.channel),
  609. total_count = item.total_count,
  610. success_count = item.success_count,
  611. fail_count = Math.Max(0, item.total_count - item.success_count)
  612. }));
  613. }
  614. if (deeplinkTableExists)
  615. {
  616. var deeplinkCounts = SqlMapper.Query<DeeplinkReportNamedSourceRow>(
  617. connection,
  618. $@"
  619. SELECT
  620. COALESCE(end_point, '') end_point,
  621. COALESCE(NULLIF(TRIM(channel_name), ''), 'unknown') channel_name,
  622. COUNT(*) total_count,
  623. CAST(COALESCE(SUM(CASE WHEN success = 1 THEN 1 ELSE 0 END), 0) AS SIGNED) success_count
  624. FROM {deeplinkTableName}
  625. GROUP BY COALESCE(end_point, ''), COALESCE(NULLIF(TRIM(channel_name), ''), 'unknown')",
  626. commandTimeout: commandTimeoutSeconds
  627. );
  628. sourceCounts.AddRange(deeplinkCounts.Select(item => new DeeplinkReportRepairSourceCount
  629. {
  630. source = deeplinkTableName,
  631. source_endpoint = item.end_point?.Trim() ?? string.Empty,
  632. channel_name = NormalizeDeeplinkReportChannelName(item.channel_name),
  633. total_count = item.total_count,
  634. success_count = item.success_count,
  635. fail_count = Math.Max(0, item.total_count - item.success_count)
  636. }));
  637. }
  638. var mergedCounts = new Dictionary<string, DeeplinkReportRepairCount>(StringComparer.OrdinalIgnoreCase);
  639. var remappedSources = new List<object>();
  640. foreach (var sourceCount in sourceCounts)
  641. {
  642. bool endpointMatched = endpointNames.TryGetValue(sourceCount.source_endpoint, out var endpointName);
  643. endpointName ??= fallbackNode.name;
  644. if (!endpointMatched)
  645. {
  646. remappedSources.Add(new
  647. {
  648. sourceCount.source,
  649. sourceEndpoint = sourceCount.source_endpoint,
  650. mappedEndpoint = endpointName,
  651. sourceCount.channel_name,
  652. sourceCount.total_count
  653. });
  654. }
  655. string mergedKey = $"{endpointName}\u001f{sourceCount.channel_name}";
  656. if (!mergedCounts.TryGetValue(mergedKey, out var mergedCount))
  657. {
  658. mergedCount = new DeeplinkReportRepairCount
  659. {
  660. endpoint_name = endpointName,
  661. channel_name = sourceCount.channel_name
  662. };
  663. mergedCounts[mergedKey] = mergedCount;
  664. }
  665. mergedCount.total_count += sourceCount.total_count;
  666. mergedCount.success_count += sourceCount.success_count;
  667. mergedCount.fail_count += sourceCount.fail_count;
  668. knownChannelNames.Add(sourceCount.channel_name);
  669. }
  670. var dateCounts = mergedCounts.Values
  671. .OrderBy(item => item.endpoint_name, StringComparer.OrdinalIgnoreCase)
  672. .ThenByDescending(item => item.total_count)
  673. .ThenBy(item => item.channel_name, StringComparer.OrdinalIgnoreCase)
  674. .ToList();
  675. var repairDate = new DeeplinkReportRepairDate
  676. {
  677. report_date = reportDate,
  678. tool_table_exists = toolTableExists,
  679. deeplink_table_exists = deeplinkTableExists,
  680. counts = dateCounts,
  681. remapped_sources = remappedSources
  682. };
  683. repairDates.Add(repairDate);
  684. }
  685. catch (Exception ex)
  686. {
  687. errors.Add(new
  688. {
  689. scope = "mysql_aggregate",
  690. date = reportDate.ToString("yyyy-MM-dd"),
  691. error = FormatRepairError(ex)
  692. });
  693. }
  694. }
  695. var redisReports = new List<object>();
  696. foreach (var endpoint in endpoints)
  697. {
  698. var endpointReports = new List<object>();
  699. string redisServer = EndPointCore.GetRedisServer(endpoint);
  700. try
  701. {
  702. await using var scope = await RedisClientFactory.CreateScopeAsync(redisServer);
  703. var redis = scope.Client;
  704. foreach (var repairDate in repairDates)
  705. {
  706. string dateKey = repairDate.report_date.ToString("yyyyMMdd");
  707. var endpointCounts = repairDate.counts
  708. .Where(item => item.endpoint_name.Equals(endpoint.name, StringComparison.OrdinalIgnoreCase))
  709. .ToList();
  710. var expectedParentCount = SumDeeplinkReportRepairCounts(endpointCounts);
  711. var beforeParentCount = await ReadDeeplinkReportBucketAsync(redis, "tool", dateKey);
  712. string channelIndexKey = $":parse_total:tool:channels:{dateKey}";
  713. var indexedChannelNames = await redis.SMembersAsync<string>(channelIndexKey) ?? [];
  714. var channelsToClear = new HashSet<string>(knownChannelNames, StringComparer.OrdinalIgnoreCase);
  715. channelsToClear.UnionWith(indexedChannelNames);
  716. channelsToClear.UnionWith(repairDate.counts
  717. .Select(item => item.channel_name)
  718. .Where(item => !string.IsNullOrWhiteSpace(item)));
  719. if (!dryRun)
  720. {
  721. await redis.DelAsync(channelIndexKey);
  722. foreach (var channelToClear in channelsToClear)
  723. {
  724. await DeleteDeeplinkReportBucketAsync(
  725. redis,
  726. $"tool:channel:{channelToClear}",
  727. dateKey
  728. );
  729. }
  730. await WriteDeeplinkReportBucketAsync(redis, "tool", dateKey, expectedParentCount);
  731. foreach (var endpointCount in endpointCounts)
  732. {
  733. await WriteDeeplinkReportBucketAsync(
  734. redis,
  735. $"tool:channel:{endpointCount.channel_name}",
  736. dateKey,
  737. endpointCount
  738. );
  739. }
  740. var endpointChannelNames = endpointCounts
  741. .Where(item => item.total_count > 0)
  742. .Select(item => item.channel_name)
  743. .Distinct(StringComparer.OrdinalIgnoreCase)
  744. .ToArray();
  745. if (endpointChannelNames.Length > 0)
  746. {
  747. await redis.SAddAsync(channelIndexKey, endpointChannelNames);
  748. await redis.ExpireAsync(channelIndexKey, DeeplinkReportStatsExpireSeconds);
  749. }
  750. }
  751. var afterParentCount = dryRun
  752. ? null
  753. : await ReadDeeplinkReportBucketAsync(redis, "tool", dateKey);
  754. endpointReports.Add(new
  755. {
  756. date = repairDate.report_date.ToString("yyyy-MM-dd"),
  757. before = beforeParentCount,
  758. expected = expectedParentCount,
  759. after = afterParentCount,
  760. channels = endpointCounts,
  761. clearedChannelCount = channelsToClear.Count
  762. });
  763. }
  764. }
  765. catch (Exception ex)
  766. {
  767. errors.Add(new
  768. {
  769. scope = "redis_write",
  770. endpoint = endpoint.name,
  771. error = FormatRepairError(ex)
  772. });
  773. }
  774. redisReports.Add(new
  775. {
  776. endpoint = endpoint.name,
  777. endpoint.description,
  778. reports = endpointReports
  779. });
  780. }
  781. var dailyReports = repairDates.Select(item => new
  782. {
  783. date = item.report_date.ToString("yyyy-MM-dd"),
  784. item.tool_table_exists,
  785. item.deeplink_table_exists,
  786. source = new
  787. {
  788. total_count = item.counts.Sum(count => count.total_count),
  789. success_count = item.counts.Sum(count => count.success_count),
  790. fail_count = item.counts.Sum(count => count.fail_count)
  791. },
  792. channels = item.counts
  793. .GroupBy(count => count.channel_name, StringComparer.OrdinalIgnoreCase)
  794. .Select(group => new
  795. {
  796. channel_name = group.Key,
  797. total_count = group.Sum(count => count.total_count),
  798. success_count = group.Sum(count => count.success_count),
  799. fail_count = group.Sum(count => count.fail_count)
  800. })
  801. .OrderByDescending(count => count.total_count)
  802. .ThenBy(count => count.channel_name, StringComparer.OrdinalIgnoreCase)
  803. .ToList(),
  804. endpoint_channels = item.counts,
  805. item.remapped_sources
  806. }).ToList();
  807. return new APIResult(new
  808. {
  809. success = errors.Count == 0,
  810. dryRun,
  811. allowPartialSources,
  812. allowCurrentDate,
  813. startDate = startDate.ToString("yyyy-MM-dd"),
  814. endDate = endDate.ToString("yyyy-MM-dd"),
  815. fallbackEndpoint = fallbackNode.name,
  816. sourceDatabase = new
  817. {
  818. endpoint = usesConfiguredParseDatabase ? parseAdminEndpoint!.name : EndPointCore.CurrentEndPoint,
  819. configured = usesConfiguredParseDatabase
  820. },
  821. processedDays = repairDates.Count,
  822. skippedDays = requestedDays - repairDates.Count,
  823. dailyReports,
  824. redis = redisReports,
  825. warnings,
  826. errors
  827. });
  828. }
  829. [HttpGet]
  830. public async Task<ActionResult> RepairTkDailyAccountStats(
  831. DateTime startDate = default,
  832. DateTime endDate = default,
  833. string accountIds = "129,140",
  834. string extraNames = "搜同款_楚颜_128众杰科技,搜同款_广哲2",
  835. bool dryRun = false,
  836. bool repairDailyLogs = true,
  837. bool repairRedis = true,
  838. bool repairRedisNameKeys = true,
  839. int commandTimeoutSeconds = 600)
  840. {
  841. if (startDate == default) startDate = new DateTime(2026, 6, 16);
  842. if (endDate == default) endDate = DateTime.Now.Date;
  843. commandTimeoutSeconds = Math.Clamp(commandTimeoutSeconds, 30, 3600);
  844. startDate = startDate.Date;
  845. endDate = endDate.Date;
  846. if (endDate < startDate)
  847. {
  848. return new APIResult(new { success = false, message = "endDate 不能早于 startDate" });
  849. }
  850. var ids = (accountIds ?? string.Empty)
  851. .Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)
  852. .Select(item => int.TryParse(item, out int id) ? id : 0)
  853. .Where(id => id > 0)
  854. .Distinct()
  855. .ToArray();
  856. if (ids.Length == 0)
  857. {
  858. return new APIResult(new { success = false, message = "accountIds 不能为空" });
  859. }
  860. using var conn = CenterHub.GetOpenConnection();
  861. if (conn.State != ConnectionState.Open) conn.Open();
  862. var errors = new List<object>();
  863. var affectedNames = new HashSet<string>(StringComparer.Ordinal);
  864. var accounts = SqlMapper.Query<TkDailyRepairCountRow>(
  865. conn,
  866. "SELECT id accountId, company accountName FROM tk_pool WHERE id IN @ids",
  867. new { ids },
  868. commandTimeout: commandTimeoutSeconds).ToList();
  869. foreach (string name in accounts.Select(item => item.accountName).Where(item => !string.IsNullOrWhiteSpace(item)))
  870. {
  871. affectedNames.Add(name);
  872. }
  873. extraNames ??= string.Empty;
  874. foreach (string name in extraNames
  875. .Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)
  876. .Where(item => !string.IsNullOrWhiteSpace(item)))
  877. {
  878. affectedNames.Add(name);
  879. }
  880. foreach (string name in SqlMapper.Query<string>(
  881. conn,
  882. @"
  883. SELECT DISTINCT accountName
  884. FROM center_daily_logs
  885. WHERE channel = 0
  886. AND accountId IN @ids
  887. AND log_date BETWEEN @startDate AND @endDate
  888. AND COALESCE(accountName,'')<>''",
  889. new { ids, startDate, endDate },
  890. commandTimeout: commandTimeoutSeconds))
  891. {
  892. affectedNames.Add(name);
  893. }
  894. var days = EachDay(startDate, endDate).ToList();
  895. var dailyReports = new List<object>();
  896. var endpointCountsByDate = new Dictionary<string, List<TkDailyRepairEndpointCountRow>>();
  897. foreach (var date in days)
  898. {
  899. try
  900. {
  901. string tableName = $"tk_parse_logs_{date:yyyyMMdd}";
  902. if (!TableExists(conn, tableName, commandTimeoutSeconds))
  903. {
  904. dailyReports.Add(new { date = date.ToString("yyyy-MM-dd"), tableName, skipped = true, reason = "table not exists" });
  905. continue;
  906. }
  907. var endpointCounts = SqlMapper.Query<TkDailyRepairEndpointCountRow>(
  908. conn,
  909. $@"
  910. SELECT
  911. l.accountId,
  912. COALESCE(p.company, MAX(l.accountName), '') accountName,
  913. COALESCE(l.end_point, '') end_point,
  914. COUNT(*) total_count,
  915. CAST(COALESCE(SUM(l.success = 1), 0) AS SIGNED) success_count,
  916. CAST(COALESCE(SUM(l.success = 0), 0) AS SIGNED) fail_count,
  917. CAST(COALESCE(SUM(l.message = '放弃转链' OR l.reason = '放弃转链'), 0) AS SIGNED) abandon_count
  918. FROM {tableName} l
  919. LEFT JOIN tk_pool p ON p.id = l.accountId
  920. WHERE l.accountId IN @ids
  921. GROUP BY l.accountId, p.company, l.end_point
  922. ORDER BY l.accountId, l.end_point",
  923. new { ids },
  924. commandTimeout: commandTimeoutSeconds).ToList();
  925. endpointCountsByDate[date.ToString("yyyyMMdd")] = endpointCounts;
  926. var accountCounts = endpointCounts
  927. .GroupBy(item => item.accountId)
  928. .Select(group => new TkDailyRepairCountRow
  929. {
  930. accountId = group.Key,
  931. accountName = accounts.FirstOrDefault(item => item.accountId == group.Key)?.accountName
  932. ?? group.FirstOrDefault()?.accountName
  933. ?? string.Empty,
  934. total_count = group.Sum(item => item.total_count),
  935. success_count = group.Sum(item => item.success_count),
  936. fail_count = group.Sum(item => item.fail_count),
  937. abandon_count = group.Sum(item => item.abandon_count)
  938. })
  939. .ToList();
  940. int dailyLogRows = 0;
  941. var allAccountCounts = ids
  942. .Select(accountId => accountCounts.FirstOrDefault(item => item.accountId == accountId)
  943. ?? new TkDailyRepairCountRow
  944. {
  945. accountId = accountId,
  946. accountName = accounts.FirstOrDefault(item => item.accountId == accountId)?.accountName ?? string.Empty
  947. })
  948. .ToList();
  949. foreach (var counts in allAccountCounts)
  950. {
  951. if (!dryRun && repairDailyLogs)
  952. {
  953. dailyLogRows += UpsertCenterDailyLog(conn, date, counts, commandTimeoutSeconds);
  954. }
  955. }
  956. dailyReports.Add(new
  957. {
  958. date = date.ToString("yyyy-MM-dd"),
  959. tableName,
  960. skipped = false,
  961. dailyLogRows,
  962. source = "mysql:tk_parse_logs_yyyyMMdd",
  963. counts = allAccountCounts,
  964. endpointCounts
  965. });
  966. }
  967. catch (Exception ex)
  968. {
  969. var error = new { scope = "daily_log", date = date.ToString("yyyy-MM-dd"), error = FormatRepairError(ex) };
  970. errors.Add(error);
  971. dailyReports.Add(new { date = date.ToString("yyyy-MM-dd"), skipped = true, reason = "error", error });
  972. }
  973. }
  974. var endpointReports = new List<object>();
  975. if (repairRedis || repairRedisNameKeys)
  976. {
  977. var endpoints = EndPointCore.List(true)
  978. .Where(node => node.status && node.is_public_api && !string.IsNullOrEmpty(EndPointCore.GetRedisServer(node)))
  979. .ToList();
  980. foreach (var endpoint in endpoints)
  981. {
  982. var redisServer = EndPointCore.GetRedisServer(endpoint);
  983. var endpointReport = new List<object>();
  984. try
  985. {
  986. await using var scope = await RedisClientFactory.CreateScopeAsync(redisServer);
  987. var redis = scope.Client;
  988. foreach (var date in days)
  989. {
  990. string dateKey = date.ToString("yyyyMMdd");
  991. if (!endpointCountsByDate.TryGetValue(dateKey, out var endpointCounts))
  992. {
  993. continue;
  994. }
  995. foreach (int accountId in ids)
  996. {
  997. var counts = endpointCounts.FirstOrDefault(item =>
  998. item.accountId == accountId &&
  999. string.Equals(item.end_point, endpoint.name, StringComparison.Ordinal))
  1000. ?? new TkDailyRepairEndpointCountRow
  1001. {
  1002. accountId = accountId,
  1003. accountName = accounts.FirstOrDefault(item => item.accountId == accountId)?.accountName ?? string.Empty,
  1004. end_point = endpoint.name
  1005. };
  1006. if (!dryRun && repairRedis)
  1007. {
  1008. await WriteParseCountKeysAsync(redis, $"tb_{accountId}", dateKey, counts);
  1009. }
  1010. endpointReport.Add(new
  1011. {
  1012. date = date.ToString("yyyy-MM-dd"),
  1013. bucket = $"tb_{accountId}",
  1014. counts
  1015. });
  1016. }
  1017. if (repairRedisNameKeys)
  1018. {
  1019. foreach (string name in affectedNames)
  1020. {
  1021. var before = await ReadParseCountKeysAsync(redis, name, dateKey);
  1022. if (!dryRun)
  1023. {
  1024. await DeleteParseBucketKeysAsync(redis, name, dateKey);
  1025. }
  1026. endpointReport.Add(new
  1027. {
  1028. date = date.ToString("yyyy-MM-dd"),
  1029. deletedBucket = name,
  1030. before
  1031. });
  1032. }
  1033. }
  1034. }
  1035. }
  1036. catch (Exception ex)
  1037. {
  1038. errors.Add(new { scope = "redis_endpoint", endpoint = endpoint.name, error = FormatRepairError(ex) });
  1039. }
  1040. endpointReports.Add(new
  1041. {
  1042. endpoint = endpoint.name,
  1043. endpoint.description,
  1044. dryRun,
  1045. deletedBuckets = endpointReport
  1046. });
  1047. }
  1048. }
  1049. return new APIResult(new
  1050. {
  1051. success = errors.Count == 0,
  1052. dryRun,
  1053. accountIds = ids,
  1054. extraNames,
  1055. repairDailyLogs,
  1056. repairRedis,
  1057. repairRedisNameKeys,
  1058. commandTimeoutSeconds,
  1059. startDate = startDate.ToString("yyyy-MM-dd"),
  1060. endDate = endDate.ToString("yyyy-MM-dd"),
  1061. affectedNames = affectedNames.OrderBy(item => item).ToList(),
  1062. mysql = dailyReports,
  1063. redis = endpointReports,
  1064. errors
  1065. });
  1066. }
  1067. private static HashSet<string> GetKnownDeeplinkReportChannelNames()
  1068. {
  1069. var channelNames = Enum.GetNames(typeof(TkChannelEnum))
  1070. .ToHashSet(StringComparer.OrdinalIgnoreCase);
  1071. try
  1072. {
  1073. var rules = new DBContext.Table("deeplink_parse_rule")
  1074. .Select<DeeplinkParseRuleDTO>() ?? Enumerable.Empty<DeeplinkParseRuleDTO>();
  1075. foreach (var rule in rules)
  1076. {
  1077. if (!string.IsNullOrWhiteSpace(rule.channel_name))
  1078. {
  1079. channelNames.Add(rule.channel_name.Trim());
  1080. }
  1081. }
  1082. }
  1083. catch
  1084. {
  1085. // The source log rows and Redis channel index still provide enough information.
  1086. }
  1087. return channelNames;
  1088. }
  1089. private static async Task<ActionResult> RepairDeeplinkReportStatsFromLegacyRedisAsync(
  1090. List<EndPointDTO> endpoints,
  1091. HashSet<string> knownChannelNames,
  1092. DateTime startDate,
  1093. DateTime endDate,
  1094. bool dryRun)
  1095. {
  1096. var sourceChannelNames = knownChannelNames
  1097. .Where(IsLegacyDeeplinkReportSourceChannel)
  1098. .OrderBy(channelName => channelName, StringComparer.OrdinalIgnoreCase)
  1099. .ToList();
  1100. var requestedDates = EachDay(startDate, endDate).ToList();
  1101. var countsByDate = requestedDates.ToDictionary(
  1102. reportDate => reportDate,
  1103. _ => new List<DeeplinkReportRepairCount>()
  1104. );
  1105. var endpointReports = new List<object>();
  1106. var warnings = new List<object>();
  1107. var errors = new List<object>();
  1108. foreach (var endpoint in endpoints)
  1109. {
  1110. var dateReports = new List<object>();
  1111. string redisServer = EndPointCore.GetRedisServer(endpoint);
  1112. try
  1113. {
  1114. await using var scope = await RedisClientFactory.CreateScopeAsync(redisServer);
  1115. var redis = scope.Client;
  1116. foreach (var reportDate in requestedDates)
  1117. {
  1118. string dateKey = reportDate.ToString("yyyyMMdd");
  1119. var readTasks = sourceChannelNames.Select(async channelName =>
  1120. {
  1121. var legacyCount = await ReadDeeplinkReportBucketAsync(redis, channelName, dateKey);
  1122. legacyCount.endpoint_name = endpoint.name;
  1123. legacyCount.channel_name = channelName;
  1124. return legacyCount;
  1125. });
  1126. var legacyCounts = (await Task.WhenAll(readTasks))
  1127. .Where(count => count.total_count > 0 || count.success_count > 0 || count.fail_count > 0)
  1128. .OrderByDescending(count => count.total_count)
  1129. .ThenBy(count => count.channel_name, StringComparer.OrdinalIgnoreCase)
  1130. .ToList();
  1131. var expectedParentCount = SumDeeplinkReportRepairCounts(legacyCounts);
  1132. var beforeParentCount = await ReadDeeplinkReportBucketAsync(redis, "tool", dateKey);
  1133. if (expectedParentCount.total_count <= 0)
  1134. {
  1135. dateReports.Add(new
  1136. {
  1137. date = reportDate.ToString("yyyy-MM-dd"),
  1138. sourceFound = false,
  1139. before = beforeParentCount,
  1140. message = "未找到旧渠道统计键,未执行清理或覆盖"
  1141. });
  1142. continue;
  1143. }
  1144. countsByDate[reportDate].AddRange(legacyCounts);
  1145. string channelIndexKey = $":parse_total:tool:channels:{dateKey}";
  1146. var indexedChannelNames = await redis.SMembersAsync<string>(channelIndexKey) ?? [];
  1147. var channelsToClear = new HashSet<string>(sourceChannelNames, StringComparer.OrdinalIgnoreCase);
  1148. channelsToClear.UnionWith(indexedChannelNames);
  1149. if (!dryRun)
  1150. {
  1151. await redis.DelAsync(channelIndexKey);
  1152. foreach (var channelToClear in channelsToClear)
  1153. {
  1154. await DeleteDeeplinkReportBucketAsync(
  1155. redis,
  1156. $"tool:channel:{channelToClear}",
  1157. dateKey
  1158. );
  1159. }
  1160. await WriteDeeplinkReportBucketAsync(redis, "tool", dateKey, expectedParentCount);
  1161. foreach (var legacyCount in legacyCounts)
  1162. {
  1163. await WriteDeeplinkReportBucketAsync(
  1164. redis,
  1165. $"tool:channel:{legacyCount.channel_name}",
  1166. dateKey,
  1167. legacyCount
  1168. );
  1169. }
  1170. string[] populatedChannelNames = legacyCounts
  1171. .Where(count => count.total_count > 0)
  1172. .Select(count => count.channel_name)
  1173. .Distinct(StringComparer.OrdinalIgnoreCase)
  1174. .ToArray();
  1175. if (populatedChannelNames.Length > 0)
  1176. {
  1177. await redis.SAddAsync(channelIndexKey, populatedChannelNames);
  1178. await redis.ExpireAsync(channelIndexKey, DeeplinkReportStatsExpireSeconds);
  1179. }
  1180. }
  1181. var afterParentCount = dryRun
  1182. ? null
  1183. : await ReadDeeplinkReportBucketAsync(redis, "tool", dateKey);
  1184. dateReports.Add(new
  1185. {
  1186. date = reportDate.ToString("yyyy-MM-dd"),
  1187. sourceFound = true,
  1188. before = beforeParentCount,
  1189. expected = expectedParentCount,
  1190. after = afterParentCount,
  1191. channels = legacyCounts,
  1192. sourceKeysPreserved = true
  1193. });
  1194. }
  1195. }
  1196. catch (Exception ex)
  1197. {
  1198. errors.Add(new
  1199. {
  1200. scope = "legacy_redis",
  1201. endpoint = endpoint.name,
  1202. error = FormatRepairError(ex)
  1203. });
  1204. }
  1205. endpointReports.Add(new
  1206. {
  1207. endpoint = endpoint.name,
  1208. endpoint.description,
  1209. reports = dateReports
  1210. });
  1211. }
  1212. foreach (var reportDate in requestedDates.Where(date => countsByDate[date].Count == 0))
  1213. {
  1214. warnings.Add(new
  1215. {
  1216. date = reportDate.ToString("yyyy-MM-dd"),
  1217. message = "所有节点均未找到旧渠道统计,不会修改该日数据"
  1218. });
  1219. }
  1220. var processedDates = requestedDates
  1221. .Where(date => countsByDate[date].Count > 0)
  1222. .ToList();
  1223. var dailyReports = processedDates.Select(reportDate =>
  1224. {
  1225. var endpointChannelCounts = countsByDate[reportDate];
  1226. var channelCounts = endpointChannelCounts
  1227. .GroupBy(count => count.channel_name, StringComparer.OrdinalIgnoreCase)
  1228. .Select(group => new
  1229. {
  1230. channel_name = group.Key,
  1231. total_count = group.Sum(count => count.total_count),
  1232. success_count = group.Sum(count => count.success_count),
  1233. fail_count = group.Sum(count => count.fail_count)
  1234. })
  1235. .OrderByDescending(count => count.total_count)
  1236. .ThenBy(count => count.channel_name, StringComparer.OrdinalIgnoreCase)
  1237. .ToList();
  1238. return new
  1239. {
  1240. date = reportDate.ToString("yyyy-MM-dd"),
  1241. total_count = channelCounts.Sum(count => count.total_count),
  1242. success_count = channelCounts.Sum(count => count.success_count),
  1243. fail_count = channelCounts.Sum(count => count.fail_count),
  1244. channels = channelCounts,
  1245. endpoint_channels = endpointChannelCounts
  1246. };
  1247. }).ToList();
  1248. return new APIResult(new
  1249. {
  1250. success = errors.Count == 0,
  1251. dryRun,
  1252. useLegacyRedis = true,
  1253. sourceMode = "legacyRedis",
  1254. sourceKeysPreserved = true,
  1255. parentTotalRule = "sum(channel totals)",
  1256. startDate = startDate.ToString("yyyy-MM-dd"),
  1257. endDate = endDate.ToString("yyyy-MM-dd"),
  1258. processedDays = processedDates.Count,
  1259. skippedDays = requestedDates.Count - processedDates.Count,
  1260. sourceChannels = sourceChannelNames,
  1261. dailyReports,
  1262. redis = endpointReports,
  1263. warnings,
  1264. errors
  1265. });
  1266. }
  1267. private static bool IsLegacyDeeplinkReportSourceChannel(string channelName)
  1268. {
  1269. if (string.IsNullOrWhiteSpace(channelName)) return false;
  1270. return !channelName.Equals("all", StringComparison.OrdinalIgnoreCase) &&
  1271. !channelName.Equals("tool", StringComparison.OrdinalIgnoreCase) &&
  1272. !channelName.Equals("unknown", StringComparison.OrdinalIgnoreCase) &&
  1273. !channelName.Equals("legacy_unclassified", StringComparison.OrdinalIgnoreCase);
  1274. }
  1275. private static string GetDeeplinkReportChannelName(int channelValue)
  1276. {
  1277. if (Enum.IsDefined(typeof(TkChannelEnum), channelValue))
  1278. {
  1279. return ((TkChannelEnum)channelValue).ToString();
  1280. }
  1281. return $"channel_{channelValue}";
  1282. }
  1283. private static string NormalizeDeeplinkReportChannelName(string channelName)
  1284. {
  1285. return string.IsNullOrWhiteSpace(channelName) ? "unknown" : channelName.Trim();
  1286. }
  1287. private static DeeplinkReportRepairCount SumDeeplinkReportRepairCounts(
  1288. IEnumerable<DeeplinkReportRepairCount> counts)
  1289. {
  1290. return new DeeplinkReportRepairCount
  1291. {
  1292. total_count = counts.Sum(item => item.total_count),
  1293. success_count = counts.Sum(item => item.success_count),
  1294. fail_count = counts.Sum(item => item.fail_count)
  1295. };
  1296. }
  1297. private static async Task<DeeplinkReportRepairCount> ReadDeeplinkReportBucketAsync(
  1298. YunhuiKit.RedisClient redis,
  1299. string bucket,
  1300. string dateKey)
  1301. {
  1302. return new DeeplinkReportRepairCount
  1303. {
  1304. total_count = await redis.GetAsync<long>($":parse_total:{bucket}:{dateKey}"),
  1305. success_count = await redis.GetAsync<long>($":parse_total:{bucket}:success:{dateKey}"),
  1306. fail_count = await redis.GetAsync<long>($":parse_total:{bucket}:fail:{dateKey}")
  1307. };
  1308. }
  1309. private static Task<long> DeleteDeeplinkReportBucketAsync(
  1310. YunhuiKit.RedisClient redis,
  1311. string bucket,
  1312. string dateKey)
  1313. {
  1314. return redis.DelAsync(
  1315. $":parse_total:{bucket}:{dateKey}",
  1316. $":parse_total:{bucket}:success:{dateKey}",
  1317. $":parse_total:{bucket}:fail:{dateKey}"
  1318. );
  1319. }
  1320. private static async Task WriteDeeplinkReportBucketAsync(
  1321. YunhuiKit.RedisClient redis,
  1322. string bucket,
  1323. string dateKey,
  1324. DeeplinkReportRepairCount counts)
  1325. {
  1326. if (counts.total_count <= 0)
  1327. {
  1328. await DeleteDeeplinkReportBucketAsync(redis, bucket, dateKey);
  1329. return;
  1330. }
  1331. bool totalWritten = await redis.SetAsync(
  1332. $":parse_total:{bucket}:{dateKey}",
  1333. counts.total_count,
  1334. DeeplinkReportStatsExpireSeconds
  1335. );
  1336. bool successWritten = await redis.SetAsync(
  1337. $":parse_total:{bucket}:success:{dateKey}",
  1338. counts.success_count,
  1339. DeeplinkReportStatsExpireSeconds
  1340. );
  1341. bool failWritten = await redis.SetAsync(
  1342. $":parse_total:{bucket}:fail:{dateKey}",
  1343. counts.fail_count,
  1344. DeeplinkReportStatsExpireSeconds
  1345. );
  1346. if (!totalWritten || !successWritten || !failWritten)
  1347. {
  1348. throw new InvalidOperationException($"Redis 写入失败:bucket={bucket}, date={dateKey}");
  1349. }
  1350. }
  1351. private sealed class DeeplinkReportToolSourceRow
  1352. {
  1353. public string end_point { get; set; } = string.Empty;
  1354. public int channel { get; set; }
  1355. public long total_count { get; set; }
  1356. public long success_count { get; set; }
  1357. }
  1358. private sealed class DeeplinkReportNamedSourceRow
  1359. {
  1360. public string end_point { get; set; } = string.Empty;
  1361. public string channel_name { get; set; } = string.Empty;
  1362. public long total_count { get; set; }
  1363. public long success_count { get; set; }
  1364. }
  1365. private sealed class DeeplinkReportRepairSourceCount
  1366. {
  1367. public string source { get; set; } = string.Empty;
  1368. public string source_endpoint { get; set; } = string.Empty;
  1369. public string channel_name { get; set; } = string.Empty;
  1370. public long total_count { get; set; }
  1371. public long success_count { get; set; }
  1372. public long fail_count { get; set; }
  1373. }
  1374. private sealed class DeeplinkReportRepairCount
  1375. {
  1376. public string endpoint_name { get; set; } = string.Empty;
  1377. public string channel_name { get; set; } = string.Empty;
  1378. public long total_count { get; set; }
  1379. public long success_count { get; set; }
  1380. public long fail_count { get; set; }
  1381. }
  1382. private sealed class DeeplinkReportRepairDate
  1383. {
  1384. public DateTime report_date { get; set; }
  1385. public bool tool_table_exists { get; set; }
  1386. public bool deeplink_table_exists { get; set; }
  1387. public List<DeeplinkReportRepairCount> counts { get; set; } = [];
  1388. public List<object> remapped_sources { get; set; } = [];
  1389. }
  1390. private static IEnumerable<DateTime> EachDay(DateTime startDate, DateTime endDate)
  1391. {
  1392. for (var date = startDate.Date; date <= endDate.Date; date = date.AddDays(1))
  1393. {
  1394. yield return date;
  1395. }
  1396. }
  1397. private static bool TableExists(IDbConnection conn, string tableName, int commandTimeoutSeconds)
  1398. {
  1399. const string sql = @"
  1400. SELECT COUNT(*)
  1401. FROM information_schema.tables
  1402. WHERE table_schema = DATABASE()
  1403. AND table_name = @tableName";
  1404. return SqlMapper.ExecuteScalar<int>(
  1405. conn,
  1406. sql,
  1407. new { tableName },
  1408. commandTimeout: commandTimeoutSeconds) > 0;
  1409. }
  1410. private static int UpsertCenterDailyLog(IDbConnection conn, DateTime date, TkDailyRepairCountRow counts, int commandTimeoutSeconds)
  1411. {
  1412. if (string.IsNullOrWhiteSpace(counts.accountName)) return 0;
  1413. string successPercentage = counts.total_count > 0 ? $"{counts.success_count / (double)counts.total_count * 100:f2}%" : string.Empty;
  1414. string abandonPercentage = counts.total_count > 0 ? $"{counts.abandon_count / (double)counts.total_count * 100:f2}%" : string.Empty;
  1415. var existingIds = SqlMapper.Query<int>(
  1416. conn,
  1417. @"
  1418. SELECT id
  1419. FROM center_daily_logs
  1420. WHERE channel = 0
  1421. AND log_date = @date
  1422. AND accountId = @accountId
  1423. ORDER BY id",
  1424. new { date, counts.accountId },
  1425. commandTimeout: commandTimeoutSeconds).ToList();
  1426. if (existingIds.Count > 0)
  1427. {
  1428. int affectedRows = SqlMapper.Execute(
  1429. conn,
  1430. @"
  1431. UPDATE center_daily_logs
  1432. SET accountName = @accountName,
  1433. parse_total_count = @totalCount,
  1434. parse_success_count = @successCount,
  1435. parse_abandon_count = @abandonCount,
  1436. parse_success_percentage = @successPercentage,
  1437. parse_abandon_percentage = @abandonPercentage,
  1438. last_time = NOW()
  1439. WHERE id = @id",
  1440. new
  1441. {
  1442. id = existingIds[0],
  1443. counts.accountName,
  1444. totalCount = counts.total_count,
  1445. successCount = counts.success_count,
  1446. abandonCount = counts.abandon_count,
  1447. successPercentage,
  1448. abandonPercentage
  1449. },
  1450. commandTimeout: commandTimeoutSeconds);
  1451. if (existingIds.Count > 1)
  1452. {
  1453. affectedRows += SqlMapper.Execute(
  1454. conn,
  1455. "DELETE FROM center_daily_logs WHERE id IN @ids",
  1456. new { ids = existingIds.Skip(1).ToArray() },
  1457. commandTimeout: commandTimeoutSeconds);
  1458. }
  1459. return affectedRows;
  1460. }
  1461. if (counts.total_count <= 0) return 0;
  1462. return SqlMapper.Execute(
  1463. conn,
  1464. @"
  1465. INSERT INTO center_daily_logs
  1466. (channel, accountId, accountName, log_date, create_time, last_time,
  1467. parse_total_count, parse_success_count, parse_abandon_count,
  1468. parse_success_percentage, parse_abandon_percentage)
  1469. VALUES
  1470. (0, @accountId, @accountName, @date, NOW(), NOW(),
  1471. @totalCount, @successCount, @abandonCount,
  1472. @successPercentage, @abandonPercentage)",
  1473. new
  1474. {
  1475. date,
  1476. counts.accountId,
  1477. counts.accountName,
  1478. totalCount = counts.total_count,
  1479. successCount = counts.success_count,
  1480. abandonCount = counts.abandon_count,
  1481. successPercentage,
  1482. abandonPercentage
  1483. },
  1484. commandTimeout: commandTimeoutSeconds);
  1485. }
  1486. private static async Task<TkDailyRepairCountRow> ReadParseCountKeysAsync(YunhuiKit.RedisClient redis, string bucket, string dateKey)
  1487. {
  1488. return new TkDailyRepairCountRow
  1489. {
  1490. accountName = bucket,
  1491. total_count = await redis.GetAsync<int>($":parse_total:{bucket}:{dateKey}"),
  1492. success_count = await redis.GetAsync<int>($":parse_total:{bucket}:success:{dateKey}"),
  1493. fail_count = await redis.GetAsync<int>($":parse_total:{bucket}:fail:{dateKey}"),
  1494. abandon_count = await redis.GetAsync<int>($":parse_total:{bucket}:放弃转链:{dateKey}")
  1495. };
  1496. }
  1497. private static async Task DeleteParseBucketKeysAsync(YunhuiKit.RedisClient redis, string bucket, string dateKey)
  1498. {
  1499. var keys = new List<string>
  1500. {
  1501. $":parse_total:{bucket}:{dateKey}",
  1502. $":parse_total:{bucket}:success:{dateKey}",
  1503. $":parse_total:{bucket}:fail:{dateKey}",
  1504. $":parse_total:{bucket}:放弃转链:{dateKey}",
  1505. $":parse_total:{bucket}:message:{dateKey}",
  1506. $":parse_total:{bucket}:reason:{dateKey}"
  1507. };
  1508. foreach (string dpBucket in new[] { "dp_none", "dp_home", "dp_success", "dp_fail" })
  1509. {
  1510. keys.Add($":parse_total:{dpBucket}:{bucket}:{dateKey}");
  1511. keys.Add($":parse_total:{dpBucket}:{bucket}:success:{dateKey}");
  1512. keys.Add($":parse_total:{dpBucket}:{bucket}:fail:{dateKey}");
  1513. keys.Add($":parse_total:{dpBucket}:{bucket}:放弃转链:{dateKey}");
  1514. keys.Add($":parse_total:{dpBucket}:{bucket}:message:{dateKey}");
  1515. keys.Add($":parse_total:{dpBucket}:{bucket}:reason:{dateKey}");
  1516. }
  1517. await redis.DelAsync(keys.ToArray());
  1518. }
  1519. private static string FormatRepairError(Exception ex)
  1520. {
  1521. return ex.InnerException == null ? ex.Message : $"{ex.Message} | {ex.InnerException.Message}";
  1522. }
  1523. private static async Task WriteParseCountKeysAsync(YunhuiKit.RedisClient redis, string bucket, string dateKey, TkDailyRepairCountRow counts)
  1524. {
  1525. var keys = new[]
  1526. {
  1527. $":parse_total:{bucket}:{dateKey}",
  1528. $":parse_total:{bucket}:success:{dateKey}",
  1529. $":parse_total:{bucket}:fail:{dateKey}",
  1530. $":parse_total:{bucket}:放弃转链:{dateKey}"
  1531. };
  1532. if (counts.total_count <= 0)
  1533. {
  1534. await redis.DelAsync(keys);
  1535. return;
  1536. }
  1537. await redis.SetAsync(keys[0], counts.total_count, 90 * 86400);
  1538. await redis.SetAsync(keys[1], counts.success_count, 90 * 86400);
  1539. await redis.SetAsync(keys[2], counts.fail_count, 90 * 86400);
  1540. await redis.SetAsync(keys[3], counts.abandon_count, 90 * 86400);
  1541. }
  1542. private class TkDailyRepairCountRow
  1543. {
  1544. public int accountId { get; set; }
  1545. public string accountName { get; set; } = string.Empty;
  1546. public long total_count { get; set; }
  1547. public long success_count { get; set; }
  1548. public long fail_count { get; set; }
  1549. public long abandon_count { get; set; }
  1550. }
  1551. private sealed class TkDailyRepairEndpointCountRow : TkDailyRepairCountRow
  1552. {
  1553. public string end_point { get; set; } = string.Empty;
  1554. }
  1555. private static async Task<string> SaveJdStressStartLogAsync(string runId, string logName, int t, int time, string ip, string oaid, bool wait)
  1556. {
  1557. try
  1558. {
  1559. await new LoggerLibrary("jd_api_stress", logName)
  1560. .AppendLine($"status=started")
  1561. .AppendLine($"runId={runId}")
  1562. .AppendLine($"startedAt={DateTime.Now:O}")
  1563. .AppendLine($"t={t}")
  1564. .AppendLine($"timeSeconds={time}")
  1565. .AppendLine($"ip={ip}")
  1566. .AppendLine($"oaid={oaid}")
  1567. .AppendLine($"wait={wait}")
  1568. .AppendLine($"content={JdStressDeeplinkContent}")
  1569. .SaveAsync();
  1570. return "started";
  1571. }
  1572. catch (Exception ex)
  1573. {
  1574. return ex.Message;
  1575. }
  1576. }
  1577. private static async Task<JdStressRunResult> RunJdStressAsync(string runId, string logName, int t, int time, string ip, string oaid, string startLogStatus, CancellationToken cancellationToken)
  1578. {
  1579. var startedAt = DateTime.Now;
  1580. var stopwatch = Stopwatch.StartNew();
  1581. long scheduledCount = 0;
  1582. long completedCount = 0;
  1583. long successCount = 0;
  1584. long failCount = 0;
  1585. long exceptionCount = 0;
  1586. long riskControlCount = 0;
  1587. long totalElapsedMs = 0;
  1588. int activeCount = 0;
  1589. int peakActiveCount = 0;
  1590. var responseLines = new ConcurrentQueue<string>();
  1591. var messageCounts = new ConcurrentDictionary<string, int>(StringComparer.Ordinal);
  1592. var codeCounts = new ConcurrentDictionary<string, int>(StringComparer.Ordinal);
  1593. int initialTaskCapacity = (int)Math.Min((long)t * Math.Min(time, 60), 4096L);
  1594. var runningTasks = new List<Task>(initialTaskCapacity);
  1595. string message = "ok";
  1596. try
  1597. {
  1598. for (int second = 0; second < time && !cancellationToken.IsCancellationRequested; second++)
  1599. {
  1600. for (int i = 0; i < t; i++)
  1601. {
  1602. long sequence = Interlocked.Increment(ref scheduledCount);
  1603. runningTasks.Add(Task.Run(async () =>
  1604. {
  1605. int currentActive = Interlocked.Increment(ref activeCount);
  1606. UpdateMax(ref peakActiveCount, currentActive);
  1607. try
  1608. {
  1609. var item = await ExecuteJdStressRequestAsync(sequence, ip, oaid, cancellationToken);
  1610. Interlocked.Increment(ref completedCount);
  1611. Interlocked.Add(ref totalElapsedMs, item.ElapsedMs);
  1612. if (item.Success)
  1613. {
  1614. Interlocked.Increment(ref successCount);
  1615. }
  1616. else
  1617. {
  1618. Interlocked.Increment(ref failCount);
  1619. }
  1620. if (item.IsException)
  1621. {
  1622. Interlocked.Increment(ref exceptionCount);
  1623. }
  1624. if (IsRiskControlResult(item))
  1625. {
  1626. Interlocked.Increment(ref riskControlCount);
  1627. }
  1628. string messageKey = string.IsNullOrWhiteSpace(item.SubMessage)
  1629. ? item.Message
  1630. : $"{item.Message}:{item.SubMessage}";
  1631. if (string.IsNullOrWhiteSpace(messageKey)) messageKey = "empty";
  1632. messageCounts.AddOrUpdate(messageKey, 1, (_, value) => value + 1);
  1633. codeCounts.AddOrUpdate(item.Code.ToString(), 1, (_, value) => value + 1);
  1634. responseLines.Enqueue(
  1635. $"{item.FinishedAt:O}\tseq={item.Sequence}\telapsedMs={item.ElapsedMs}\tcode={item.Code}\tsuccess={item.Success}\tmessage={NormalizeLogValue(item.Message)}\tsub_message={NormalizeLogValue(item.SubMessage)}\tresponse={NormalizeLogValue(item.Response)}");
  1636. }
  1637. finally
  1638. {
  1639. Interlocked.Decrement(ref activeCount);
  1640. }
  1641. }));
  1642. }
  1643. await ObserveCompletedTasksAsync(runningTasks);
  1644. var nextTick = TimeSpan.FromSeconds(second + 1);
  1645. var delay = nextTick - stopwatch.Elapsed;
  1646. if (delay > TimeSpan.Zero)
  1647. {
  1648. await Task.Delay(delay, cancellationToken);
  1649. }
  1650. }
  1651. }
  1652. catch (OperationCanceledException ex)
  1653. {
  1654. message = $"canceled:{ex.Message}";
  1655. responseLines.Enqueue($"{DateTime.Now:O}\tmessage={NormalizeLogValue(message)}");
  1656. }
  1657. catch (Exception ex)
  1658. {
  1659. message = $"runner error:{ex.Message}";
  1660. responseLines.Enqueue($"{DateTime.Now:O}\tmessage={NormalizeLogValue(message)}\tstack={NormalizeLogValue(ex.StackTrace ?? string.Empty)}");
  1661. }
  1662. try
  1663. {
  1664. await Task.WhenAll(runningTasks);
  1665. }
  1666. catch (Exception ex)
  1667. {
  1668. message = $"task wait error:{ex.Message}";
  1669. responseLines.Enqueue($"{DateTime.Now:O}\tmessage={NormalizeLogValue(message)}\tstack={NormalizeLogValue(ex.StackTrace ?? string.Empty)}");
  1670. }
  1671. stopwatch.Stop();
  1672. string logStatus = "saved";
  1673. try
  1674. {
  1675. var log = new LoggerLibrary("jd_api_stress", logName);
  1676. log.AppendLine($"status=finished");
  1677. log.AppendLine($"startLogStatus={startLogStatus}");
  1678. log.AppendLine($"runId={runId}");
  1679. log.AppendLine($"startedAt={startedAt:O}");
  1680. log.AppendLine($"finishedAt={DateTime.Now:O}");
  1681. log.AppendLine($"t={t}");
  1682. log.AppendLine($"timeSeconds={time}");
  1683. log.AppendLine($"ip={ip}");
  1684. log.AppendLine($"oaid={oaid}");
  1685. log.AppendLine($"content={JdStressDeeplinkContent}");
  1686. log.AppendLine($"scheduled={scheduledCount}");
  1687. log.AppendLine($"completed={completedCount}");
  1688. log.AppendLine($"success={successCount}");
  1689. log.AppendLine($"fail={failCount}");
  1690. log.AppendLine($"exception={exceptionCount}");
  1691. log.AppendLine($"riskControl={riskControlCount}");
  1692. log.AppendLine($"peakActive={peakActiveCount}");
  1693. log.AppendLine("messageCounts=");
  1694. foreach (var item in messageCounts.OrderByDescending(item => item.Value))
  1695. {
  1696. log.AppendLine($"{item.Key}\t{item.Value}");
  1697. }
  1698. log.AppendLine("codeCounts=");
  1699. foreach (var item in codeCounts.OrderByDescending(item => item.Value))
  1700. {
  1701. log.AppendLine($"{item.Key}\t{item.Value}");
  1702. }
  1703. log.AppendLine("responses=");
  1704. while (responseLines.TryDequeue(out string? line))
  1705. {
  1706. log.AppendLine(line);
  1707. }
  1708. await log.SaveAsync();
  1709. }
  1710. catch (Exception ex)
  1711. {
  1712. logStatus = ex.Message;
  1713. }
  1714. return new JdStressRunResult
  1715. {
  1716. success = true,
  1717. message = message,
  1718. runId = runId,
  1719. requestPerSecond = t,
  1720. timeSeconds = time,
  1721. expectedRequests = (long)t * time,
  1722. scheduledCount = scheduledCount,
  1723. completedCount = completedCount,
  1724. successCount = successCount,
  1725. failCount = failCount,
  1726. exceptionCount = exceptionCount,
  1727. riskControlCount = riskControlCount,
  1728. peakActiveCount = peakActiveCount,
  1729. averageElapsedMs = completedCount == 0 ? 0 : Math.Round(totalElapsedMs / (double)completedCount, 2),
  1730. elapsedSeconds = Math.Round(stopwatch.Elapsed.TotalSeconds, 2),
  1731. messageCounts = messageCounts.OrderByDescending(item => item.Value).ToDictionary(item => item.Key, item => item.Value),
  1732. codeCounts = codeCounts.OrderByDescending(item => item.Value).ToDictionary(item => item.Key, item => item.Value),
  1733. log = new JdStressLogInfo
  1734. {
  1735. type = "LoggerLibrary",
  1736. dir = "jd_api_stress",
  1737. name = logName,
  1738. status = logStatus
  1739. }
  1740. };
  1741. }
  1742. private static async Task ObserveCompletedTasksAsync(List<Task> runningTasks)
  1743. {
  1744. for (int i = runningTasks.Count - 1; i >= 0; i--)
  1745. {
  1746. if (!runningTasks[i].IsCompleted) continue;
  1747. await runningTasks[i];
  1748. runningTasks.RemoveAt(i);
  1749. }
  1750. }
  1751. private static async Task<JdStressRequestResult> ExecuteJdStressRequestAsync(long sequence, string ip, string oaid, CancellationToken cancellationToken)
  1752. {
  1753. var sw = Stopwatch.StartNew();
  1754. try
  1755. {
  1756. var request = new UnionParseRequest
  1757. {
  1758. Content = JdStressDeeplinkContent,
  1759. Channel = "jd",
  1760. CommerceType = 0,
  1761. Ip = ip,
  1762. Oaid = oaid,
  1763. RiskStrategy = string.Empty,
  1764. LaunchScene = 0,
  1765. AccountId = 0,
  1766. SpecialText = 0,
  1767. QueryText = string.Empty,
  1768. ClickId = string.Empty,
  1769. Type = "dp",
  1770. Pic = string.Empty
  1771. };
  1772. var result = await UnionParseCore.DeeplinkJdParseAsync(request, cancellationToken);
  1773. sw.Stop();
  1774. string response = result.Content ?? string.Empty;
  1775. var parsed = ParseJdStressResponse(response);
  1776. parsed.Sequence = sequence;
  1777. parsed.ElapsedMs = sw.ElapsedMilliseconds;
  1778. parsed.Response = response;
  1779. parsed.FinishedAt = DateTime.Now;
  1780. return parsed;
  1781. }
  1782. catch (Exception ex)
  1783. {
  1784. sw.Stop();
  1785. return new JdStressRequestResult
  1786. {
  1787. Sequence = sequence,
  1788. ElapsedMs = sw.ElapsedMilliseconds,
  1789. Success = false,
  1790. Message = "exception",
  1791. SubMessage = ex.Message,
  1792. Code = 0,
  1793. Response = ex.ToString(),
  1794. FinishedAt = DateTime.Now,
  1795. IsException = true
  1796. };
  1797. }
  1798. }
  1799. private static JdStressRequestResult ParseJdStressResponse(string response)
  1800. {
  1801. var result = new JdStressRequestResult();
  1802. if (string.IsNullOrWhiteSpace(response))
  1803. {
  1804. result.Message = "empty response";
  1805. return result;
  1806. }
  1807. try
  1808. {
  1809. using var doc = JsonDocument.Parse(response);
  1810. var root = doc.RootElement;
  1811. result.Success = ReadBool(root, "success");
  1812. result.Message = ReadString(root, "message");
  1813. result.SubMessage = ReadString(root, "sub_message");
  1814. result.Code = ReadInt(root, "code");
  1815. }
  1816. catch (Exception ex)
  1817. {
  1818. result.Success = false;
  1819. result.Message = "parse response error";
  1820. result.SubMessage = ex.Message;
  1821. result.Code = 0;
  1822. }
  1823. return result;
  1824. }
  1825. private static bool ReadBool(JsonElement root, string propertyName)
  1826. {
  1827. if (!root.TryGetProperty(propertyName, out var property)) return false;
  1828. return property.ValueKind switch
  1829. {
  1830. JsonValueKind.True => true,
  1831. JsonValueKind.False => false,
  1832. JsonValueKind.Number => property.TryGetInt32(out int value) && value != 0,
  1833. JsonValueKind.String => bool.TryParse(property.GetString(), out bool value) && value,
  1834. _ => false
  1835. };
  1836. }
  1837. private static int ReadInt(JsonElement root, string propertyName)
  1838. {
  1839. if (!root.TryGetProperty(propertyName, out var property)) return 0;
  1840. return property.ValueKind switch
  1841. {
  1842. JsonValueKind.Number => property.TryGetInt32(out int value) ? value : 0,
  1843. JsonValueKind.String => int.TryParse(property.GetString(), out int value) ? value : 0,
  1844. _ => 0
  1845. };
  1846. }
  1847. private static string ReadString(JsonElement root, string propertyName)
  1848. {
  1849. if (!root.TryGetProperty(propertyName, out var property)) return string.Empty;
  1850. if (property.ValueKind == JsonValueKind.Null || property.ValueKind == JsonValueKind.Undefined) return string.Empty;
  1851. return property.ValueKind == JsonValueKind.String ? property.GetString() ?? string.Empty : property.ToString();
  1852. }
  1853. private static bool IsRiskControlResult(JdStressRequestResult result)
  1854. {
  1855. if (!"放弃转链".Equals(result.Message, StringComparison.Ordinal)) return false;
  1856. return result.SubMessage.Contains("控制", StringComparison.Ordinal)
  1857. || result.SubMessage.Contains("风控", StringComparison.Ordinal)
  1858. || result.SubMessage.Contains("限流", StringComparison.Ordinal)
  1859. || result.SubMessage.Contains("频", StringComparison.Ordinal)
  1860. || result.SubMessage.Contains("系统繁忙", StringComparison.Ordinal);
  1861. }
  1862. private static string NormalizeLogValue(string value)
  1863. {
  1864. return (value ?? string.Empty)
  1865. .Replace("\r", "\\r")
  1866. .Replace("\n", "\\n")
  1867. .Replace("\t", " ");
  1868. }
  1869. private static void UpdateMax(ref int target, int value)
  1870. {
  1871. int snapshot;
  1872. while (value > (snapshot = Volatile.Read(ref target))
  1873. && Interlocked.CompareExchange(ref target, value, snapshot) != snapshot)
  1874. {
  1875. }
  1876. }
  1877. private sealed class JdStressRequestResult
  1878. {
  1879. public long Sequence { get; set; }
  1880. public bool Success { get; set; }
  1881. public string Message { get; set; } = string.Empty;
  1882. public string SubMessage { get; set; } = string.Empty;
  1883. public int Code { get; set; }
  1884. public long ElapsedMs { get; set; }
  1885. public string Response { get; set; } = string.Empty;
  1886. public DateTime FinishedAt { get; set; }
  1887. public bool IsException { get; set; }
  1888. }
  1889. private sealed class JdStressRunResult
  1890. {
  1891. public bool success { get; set; }
  1892. public string message { get; set; } = string.Empty;
  1893. public string runId { get; set; } = string.Empty;
  1894. public int requestPerSecond { get; set; }
  1895. public int timeSeconds { get; set; }
  1896. public long expectedRequests { get; set; }
  1897. public long scheduledCount { get; set; }
  1898. public long completedCount { get; set; }
  1899. public long successCount { get; set; }
  1900. public long failCount { get; set; }
  1901. public long exceptionCount { get; set; }
  1902. public long riskControlCount { get; set; }
  1903. public int peakActiveCount { get; set; }
  1904. public double averageElapsedMs { get; set; }
  1905. public double elapsedSeconds { get; set; }
  1906. public Dictionary<string, int> messageCounts { get; set; } = [];
  1907. public Dictionary<string, int> codeCounts { get; set; } = [];
  1908. public JdStressLogInfo log { get; set; } = new();
  1909. }
  1910. private sealed class JdStressLogInfo
  1911. {
  1912. public string type { get; set; } = string.Empty;
  1913. public string dir { get; set; } = string.Empty;
  1914. public string name { get; set; } = string.Empty;
  1915. public string status { get; set; } = string.Empty;
  1916. }
  1917. [HttpGet]
  1918. public async Task<ActionResult> xxx()
  1919. {
  1920. var list = await TkPoolCore.ListAsync();
  1921. if (list == null) return new APIResult(new { success = false, message = "没有有效账号", });
  1922. string message = string.Empty;
  1923. foreach (var account in list)
  1924. {
  1925. try
  1926. {
  1927. var alimama = new AlimamaPlus(account);
  1928. alimama.RenewCookie();
  1929. }
  1930. catch (Exception ex)
  1931. {
  1932. message = $"【cookie续期】xxxx\n{ex.Message}\n{ex.StackTrace}";
  1933. NotifyCore.Notify(new NifyMessage
  1934. {
  1935. message = message,
  1936. priority = NifyMessagePriority.high,
  1937. tags = ["red_circle"]
  1938. });
  1939. continue;
  1940. }
  1941. }
  1942. return new APIResult(new { success = true, message = "ok" });
  1943. }
  1944. }
  1945. }