| 1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512 |
- using molilian.core;
- using dodohold.core;
- using Dapper;
- using Microsoft.AspNetCore.Mvc;
- using Org.BouncyCastle.Ocsp;
- using System.Collections.Concurrent;
- using System.Diagnostics;
- using System.Text.Json;
- using System.Runtime.InteropServices;
- using System.Net;
- using System.Threading;
- using System.Data;
- using Microsoft.AspNetCore.Mvc.RazorPages;
- using TencentCloud.Soe.V20180724.Models;
- using static dodohold.core.ZTOExpress.CreateOrderArgs;
- using static Spire.Xls.Core.Spreadsheet.HTMLOptions;
- using System.Net.Http.Json;
- using System.Text.RegularExpressions;
- using System.Text;
- using TencentCloud.Oceanus.V20190422.Models;
- using YunhuiKit;
- using TencentCloud.Cdn.V20180606.Models;
- namespace molilian.api.Controllers
- {
- [ApiController]
- [Route("[controller]/[action]")]
- public class TestController : ControllerBase
- {
- private const string JdStressDeeplinkContent = "openapp.jdmobile://virtual?params=%7B%22category%22%3A%22jump%22%2C%22des%22%3A%22productDetail%22%2C%22skuId%22%3A%2210139003162781%22%2C%22sourceType%22%3A%22Item%22%2C%22sourceValue%22%3A%22view-ware%22%7D";
- private const int DeeplinkReportStatsExpireSeconds = 90 * 86400;
- private const int DeeplinkReportRepairMaxDays = 366;
- protected IHttpContextAccessor _accessor;
- public TestController(IHttpContextAccessor accessor)
- {
- _accessor = accessor;
- }
- [HttpGet]
- public async Task<ActionResult> jd_unsafe_parse_stress([FromQuery] int t = 5, [FromQuery] int time = 60, [FromQuery] string ip = "127.0.0.1", [FromQuery] string oaid = "", [FromQuery] bool wait = false)
- {
- t = t <= 0 ? 5 : t;
- time = time <= 0 ? 60 : time;
- string runId = DateTime.Now.ToString("yyyyMMdd_HHmmss_fff");
- string logName = $"run_{runId}_t{t}_time{time}";
- string startLogStatus = await SaveJdStressStartLogAsync(runId, logName, t, time, ip, oaid, wait);
- if (wait)
- {
- var result = await RunJdStressAsync(runId, logName, t, time, ip, oaid, startLogStatus, CancellationToken.None);
- return new APIResult(result);
- }
- _ = Task.Run(async () =>
- {
- try
- {
- await RunJdStressAsync(runId, logName, t, time, ip, oaid, startLogStatus, CancellationToken.None);
- }
- catch (Exception ex)
- {
- _ = new LoggerLibrary("jd_api_stress", $"{logName}_runner_error")
- .Info(ex.Message, ex.StackTrace)
- .SaveAsync();
- }
- });
- return new APIResult(new
- {
- success = true,
- message = "后台压测已启动",
- runId,
- requestPerSecond = t,
- timeSeconds = time,
- expectedRequests = (long)t * time,
- wait,
- log = new
- {
- type = "LoggerLibrary",
- dir = "jd_api_stress",
- name = logName,
- status = startLogStatus
- }
- });
- }
- [HttpGet]
- public async Task<ActionResult> ecs_list_test()
- {
- var list = AliyunPoolCore.EcsList();
- return new APIResult(new
- {
- success = true,
- msg = "ok",
- list
- });
- }
-
- [HttpGet]
- public async Task<ActionResult> testReconnectionRedis()
- {
- string cacheKey = "test";
- RedisKit.SetAsync(cacheKey, 1, 3600);
- string val = await RedisKit.GetAsync<string>(cacheKey);
- return new APIResult(new { success = "ok", val });
- }
- [HttpGet]
- public async Task<ActionResult> backfill_track_parse_metrics([FromQuery] string reportDate = "")
- {
- DateTime targetDate = DateTime.Now.Date;
- if (!string.IsNullOrWhiteSpace(reportDate) && !DateTime.TryParse(reportDate, out targetDate))
- {
- return new APIResult(new
- {
- success = false,
- message = "reportDate格式错误,请使用 yyyy-MM-dd"
- });
- }
- var result = await TracksCore.BackfillParseMetricCountersAsync(targetDate);
- return new APIResult(new
- {
- success = true,
- message = "ok",
- data = result
- });
- }
- [HttpGet]
- public async Task<ActionResult> ChangePublicIpByName(string nodeName)
- {
- var proxy_node = new DBContext.Table("proxy_nodes").Get<dynamic>("nodeName=@nodeName", new { nodeName });
- if (proxy_node == null) return new APIResult(new { success = false, msg = "没有匹配的 proxy_nodes 记录" });
- int aliyun_id = proxy_node.aliyun_id;
- string proxy_server = proxy_node.server;
- var account = new DBContext.Table("aliyun_pool").Get<AliyunPoolDTO>(aliyun_id);
- if (account == null) return new APIResult(new { success = false, msg = "没有匹配的 aliyun_pool 记录" });
- var uri = new Uri(proxy_server);
- string privateIp = uri.Host;
- AliyunCore core = new AliyunCore(account);
- var success = await core.ChangePublicIpAsync(privateIp);
- return new APIResult(new { success = "ok" });
- }
- [HttpGet]
- public async Task<ActionResult> ChangePublicIp(int id)
- {
- id = id >= 20000 ? id - 20000 : id;
- var proxy_node = new DBContext.Table("proxy_nodes").Get<dynamic>(id);
- if (proxy_node == null) return new APIResult(new { success = false, msg = "没有匹配的 proxy_nodes 记录" });
- int aliyun_id = proxy_node.aliyun_id;
- string proxy_server = proxy_node.server;
- var account = new DBContext.Table("aliyun_pool").Get<AliyunPoolDTO>(aliyun_id);
- if (account == null) return new APIResult(new { success = false, msg = "没有匹配的 aliyun_pool 记录" });
- var uri = new Uri(proxy_server);
- string privateIp = uri.Host;
- AliyunCore core = new AliyunCore(account);
- var success = await core.ChangePublicIpAsync(privateIp);
- return new APIResult(new { success = "ok" });
- }
- [HttpGet]
- //第一部 先加一个网卡,
- public async Task<ActionResult> CreateNetworkInterface(int id, string ecsid)
- {
- AliyunCore core = new AliyunCore(id);
- //创建弹性网卡并绑定公网IP
- var success = core.EcsCreateNetworkInterface("cn-beijing", ecsid);
- return new APIResult(new { success });
- }
- [HttpGet]
- //第二部 给网卡绑定很多个辅助ip
- public async Task<ActionResult> EcsAssignPrivateIpAddresses(int id, string ecsid, int count, bool isSecondary = true)
- {
- AliyunCore core = new AliyunCore(id);
- //创建弹性网卡并绑定公网IP
- var success = core.EcsAssignPrivateIpAddresses("cn-beijing", ecsid, count, !isSecondary);
- return new APIResult(new { success });
- }
- [HttpGet]
- public async Task<ActionResult> QueryAccountBalance()
- {
- AliyunPlus plus = new AliyunPlus("LTAI5tQbkTjtULQcrWGaw2VJ", "UIkkolVVEddooOKOIUByCqymZkK6ZA");
- var response = plus.QueryAccountBalance();
- var balance = response.Body.Data.AvailableAmount;
- return new APIResult(new { response });
- }
- //[HttpGet]
- //public async Task<ActionResult> test(string table = "tk_parse_logs_shop", int count = 100)
- //{
- // //table = "tk_parse_logs_shop";
- // //table = "tk_parse_logs_live";
- // //table = "tk_parse_logs_video";
- // string filter = "success=0 AND message='放弃转链' AND reason='非标准链接'";
- // for (int i = 0; i < 100; i++)
- // {
- // try
- // {
- // var list = new DBContext.Table(table)
- // .Where(filter, new { })
- // .Limit(count).Select<TkDataDTO>();
- // if (!list.Any())
- // {
- // return new APIResult(new
- // {
- // success = false,
- // message = "所有任务完成"
- // });
- // }
- // foreach (var item in list)
- // {
- // TkPoolDTO? account = TkPoolCore.GetOne(TkPoolCore.TkAction.parse);
- // if (account == null)
- // {
- // return new APIResult(new
- // {
- // success = false,
- // message = "没有工作账号"
- // });
- // }
- // var result = item.Convert2Json().Convert2Object<TkDataDTO>();
- // result.accountId = account.id;
- // result.accountName = account.company;
- // var alimama = new AlimamaPlus(account);
- // int timeout = alimama._config.rt_max;
- // using var cts = new CancellationTokenSource();
- // cts.CancelAfter(timeout);
- // result = await alimama.UnionParseAsync(item.content, result);
- // string reason = "非标准链接".Equals(result.reason) ? "非标准链接ok" : result.reason;
- // new DBContext.Table(table)
- // .Add("linkType", (int)result.link_type)
- // .Add("rawContent", result.rawContent)
- // .Add("success", result.success)
- // .Add("message", result.message)
- // .Add("reason", reason)
- // .Add("content", result.content)
- // .Add("itemId", result.itemId)
- // .Add("itemName", result.itemName)
- // .Add("pic", result.pic)
- // .Add("couponAmount", result.couponAmount)
- // .Add("promotionPrice", result.promotionPrice)
- // .Add("taoToken", result.taoToken)
- // .Add("shortLinkurl", result.shortLinkurl)
- // .Add("deeplink_url", result.deeplink_url)
- // .Add("num_iid", result.num_iid)
- // .Add("elapsedTime", result.elapsedTime)
- // .Add("elapsedTime2", result.elapsedTime2)
- // .Add("elapsedTime3", result.elapsedTime3)
- // .Add("subCode", result.subCode)
- // .Where("id=@id", new { item.id })
- // .Update();
- // }
- // }
- // catch
- // {
- // }
- // }
- // return new APIResult(new
- // {
- // success = true,
- // message = "ok"
- // });
- //}
- [HttpGet]
- public async Task<ActionResult> redu_douyin(string command)
- {
- ReduPlus plus = new ReduPlus("", "", "");
- var result = await plus.DouyinParse(command);
- return Content(result.Convert2Json());
- }
- [HttpGet]
- public async Task<ActionResult> redu_kuaishou(string command)
- {
- ReduPlus plus = new ReduPlus("", "", "");
- var result = await plus.KuaishouParse(command);
- return Content(result.Convert2Json());
- }
- [HttpPost]
- public async Task<ActionResult> test2([FromForm] string jsonContent, [FromForm] string testText)
- {
- // 用于存储输出结果的StringBuilder
- StringBuilder outputBuilder = new StringBuilder();
- // 解析JSON数据
- JsonDocument jsonDoc = JsonDocument.Parse(jsonContent);
- // 获取根元素
- JsonElement root = jsonDoc.RootElement;
- // 遍历规则
- foreach (JsonElement ruleSet in root.EnumerateArray())
- {
- string platformType = ruleSet.Read("platformType", string.Empty);
- string supplier = ruleSet.Read("supplier", string.Empty);
- outputBuilder.AppendLine($"平台类型: {platformType}, 供应商: {supplier}");
- JsonElement pwdRules = ruleSet.GetProperty("pwdRules");
- int idx = 0;
- foreach (JsonElement patternElement in pwdRules.EnumerateArray())
- {
- string pattern = patternElement.GetString();
- try
- {
- // 使用Regex类来编译正则表达式
- Regex compiledPattern = new Regex(pattern);
- if (compiledPattern.IsMatch(testText))
- {
- outputBuilder.AppendLine($"规则 {idx + 1}: 匹配\t{pattern}");
- }
- else
- {
- outputBuilder.AppendLine($"规则 {idx + 1}: 不匹配");
- }
- }
- catch (Exception e)
- {
- outputBuilder.AppendLine($"规则 {idx + 1}: 正则表达式错误 - {e.Message}\t{pattern}");
- }
- idx++;
- }
- }
- outputBuilder.AppendLine("测试完成");
- return Content(outputBuilder.ToString());
- }
- [HttpGet]
- public async Task<ActionResult> ip(string ip)
- {
- string result = IP2RegionPlus.Search(ip);
- return new APIResult(new
- {
- success = true,
- message = result
- });
- }
- [HttpPost]
- public async Task<ActionResult> testreg([FromBody] JsonElement form)
- {
- var content = form.Read("s", string.Empty);
- string shortLinkurl = AlimamaPlus.GetTaobaoLink(content);
- bool is_tao_token = AlimamaPlus.MatchRegexes(content, []);
- bool is_other = AlimamaPlus.MatchOtherInfo(content, []);
- return new APIResult(new
- {
- shortLinkurl,
- is_tao_token,
- is_other
- });
- }
- //[HttpGet]
- //public async Task<ActionResult> comparison_tk([FromQuery] int count = 100)
- //{
- // string cacheKey = ":lock_key:comparison_tk_logs";
- // int last_id = RedisHelper.Get<int>(cacheKey);
- // string filter = "id>@last_id";
- // var result = new DBContext.Table("comparison_tk_logs")
- // .Where(filter, new { last_id })
- // .Page(count, 1)
- // .Order("ID")
- // .PageList<TkDataDTO>(false);
- // var ip = "127.0.0.1";
- // var oaid = "test-comparison_tk_logs";
- // foreach (var item in result.List)
- // {
- // await UnionParseCore.TaobaoParseAsync(item.rawContent, ip, oaid);
- // RedisHelper.Set(cacheKey, item.id, 10 * 86400);
- // }
- // if (result.Count < count)
- // {
- // cacheKey = ":lock_key:start_comparison_tk";
- // RedisHelper.Set(cacheKey, 1, 600);
- // }
- // return new APIResult(new
- // {
- // success = true,
- // message = "ok"
- // });
- //}
- //[HttpGet]
- //public async Task<ActionResult> comparison_tk_raw([FromQuery] int count = 100)
- //{
- // string cacheKey = ":lock_key:comparison_tk_logs_raw";
- // int last_id = RedisHelper.Get<int>(cacheKey);
- // string filter = "id>@last_id";
- // var result = new DBContext.Table("comparison_tk_logs")
- // .Where(filter, new { last_id })
- // .Page(count, 1)
- // .Order("ID")
- // .PageList<TkDataDTO>(false);
- // var ip = "127.0.0.1";
- // var oaid = "test-comparison_tk_logs_raw";
- // foreach (var item in result.List)
- // {
- // await UnionParseCore.TaobaoParseAsync(item.rawContent, ip, oaid);
- // RedisHelper.Set(cacheKey, item.id, 10 * 86400);
- // }
- // if (result.Count < count)
- // {
- // cacheKey = ":lock_key:start_comparison_tk";
- // RedisHelper.Set(cacheKey, 1, 600);
- // }
- // return new APIResult(new
- // {
- // success = true,
- // message = "ok"
- // });
- //}
- [HttpGet]
- public async Task<ActionResult> testTask()
- {
- string url = "https://www.taobao.com";
- TkDataDTO result = new TkDataDTO();
- result.message = "init";
- result.success = true;
- int timeout = 1000;
- using var cts = new CancellationTokenSource();
- cts.CancelAfter(timeout);
- var requestTask = Task.Run(() => AlimamaPlus.testTaskAsync(result, cts.Token), cts.Token);
- var delayTask = Task.Delay(timeout, cts.Token);
- var completedTask = await Task.WhenAny(requestTask, delayTask);
- if (completedTask == requestTask)
- {
- result = await requestTask;
- }
- else
- {
- cts.Cancel();
- result.success = false;
- result.message = "放弃转链";
- result.reason = "请求超时";
- result.itemName = "点击打开淘宝APP";
- }
- return new APIResult(new
- {
- success = true,
- message = "ok",
- result,
- });
- }
- [HttpGet]
- public async Task<ActionResult> testThread()
- {
- // 查看默认的最小和最大线程数
- ThreadPool.GetMinThreads(out int defaultMinWorker, out int defaultMinIOC);
- ThreadPool.GetMaxThreads(out int defaultMaxWorker, out int defaultMaxIOC);
- string tmp = $"Default Min worker threads: {defaultMinWorker}, Min I/O completion threads: {defaultMinIOC}";
- string tmp2 = $"Default Max worker threads: {defaultMaxWorker}, Max I/O completion threads: {defaultMaxIOC}";
- // 获取当前线程池中可用的工作线程数和 I/O 完成端口线程数
- ThreadPool.GetAvailableThreads(out int availableWorkerThreads, out int availableIOCompletionThreads);
- string tmp3 = $"Current available worker threads: {availableWorkerThreads}, available I/O completion threads: {availableIOCompletionThreads}";
- return new APIResult(new
- {
- tmp,
- tmp2,
- tmp3,
- });
- }
- [HttpPost]
- public async Task<ActionResult> test1([FromBody] JsonElement form)
- {
- NotifyCore.Notify(new NifyMessage
- {
- message = $"【淘宝联盟:test】cookie 掉线",
- priority = NifyMessagePriority.high,
- tags = ["red_circle"]
- });
- return new APIResult(new
- {
- success = true,
- message = "ok"
- });
- }
- [HttpGet]
- public async Task<ActionResult> RepairDeeplinkReportStats(
- DateTime startDate = default,
- DateTime endDate = default,
- bool dryRun = true,
- bool useLegacyRedis = false,
- bool allowPartialSources = false,
- bool allowCurrentDate = false,
- string fallbackEndpoint = "",
- int commandTimeoutSeconds = 600)
- {
- var yesterday = DateTime.Now.Date.AddDays(-1);
- if (startDate == default) startDate = yesterday;
- if (endDate == default) endDate = startDate;
- startDate = startDate.Date;
- endDate = endDate.Date;
- commandTimeoutSeconds = Math.Clamp(commandTimeoutSeconds, 30, 3600);
- if (endDate < startDate)
- {
- return new APIResult(new { success = false, message = "endDate 不能早于 startDate" });
- }
- int requestedDays = (endDate - startDate).Days + 1;
- if (requestedDays > DeeplinkReportRepairMaxDays)
- {
- return new APIResult(new
- {
- success = false,
- message = $"单次最多重建 {DeeplinkReportRepairMaxDays} 天数据"
- });
- }
- if (!allowCurrentDate && endDate >= DateTime.Now.Date)
- {
- return new APIResult(new
- {
- success = false,
- message = "默认禁止重建当天数据,避免覆盖实时计数;如确需执行请传 allowCurrentDate=true"
- });
- }
- var endpoints = (EndPointCore.List(true) ?? Enumerable.Empty<EndPointDTO>())
- .Where(node => node.status && node.is_public_api)
- .Where(node => !string.IsNullOrWhiteSpace(EndPointCore.GetRedisServer(node)))
- .GroupBy(node => node.name, StringComparer.OrdinalIgnoreCase)
- .Select(group => group.First())
- .OrderBy(node => node.name, StringComparer.OrdinalIgnoreCase)
- .ToList();
- if (endpoints.Count == 0)
- {
- return new APIResult(new { success = false, message = "没有可用的公共 API Redis 节点" });
- }
- EndPointDTO fallbackNode;
- if (!string.IsNullOrWhiteSpace(fallbackEndpoint))
- {
- fallbackNode = endpoints.FirstOrDefault(node => node.name.Equals(
- fallbackEndpoint.Trim(),
- StringComparison.OrdinalIgnoreCase
- ))!;
- if (fallbackNode == null)
- {
- return new APIResult(new
- {
- success = false,
- message = $"fallbackEndpoint={fallbackEndpoint} 不在可用公共节点中",
- availableEndpoints = endpoints.Select(node => node.name).ToList()
- });
- }
- }
- else
- {
- fallbackNode = endpoints.FirstOrDefault(node => node.name.Equals(
- EndPointCore.CurrentEndPoint,
- StringComparison.OrdinalIgnoreCase
- )) ?? endpoints[0];
- }
- var endpointNames = endpoints.ToDictionary(
- node => node.name,
- node => node.name,
- StringComparer.OrdinalIgnoreCase
- );
- var knownChannelNames = GetKnownDeeplinkReportChannelNames();
- var repairDates = new List<DeeplinkReportRepairDate>();
- var warnings = new List<object>();
- var errors = new List<object>();
- if (useLegacyRedis)
- {
- return await RepairDeeplinkReportStatsFromLegacyRedisAsync(
- endpoints,
- knownChannelNames,
- startDate,
- endDate,
- dryRun
- );
- }
- var parseAdminEndpoint = EndPointCore.GetParseAdmin();
- bool usesConfiguredParseDatabase = parseAdminEndpoint != null &&
- !string.IsNullOrWhiteSpace(parseAdminEndpoint.db_server);
- using var connection = usesConfiguredParseDatabase
- ? EndPointCore.GetDbConnection(parseAdminEndpoint!.db_server)
- : DBContext.GetOpenConnection();
- if (connection.State != ConnectionState.Open) connection.Open();
- foreach (var reportDate in EachDay(startDate, endDate))
- {
- string dateKey = reportDate.ToString("yyyyMMdd");
- string toolTableName = $"tool_parse_logs_{dateKey}";
- string deeplinkTableName = $"deeplink_parse_logs_{dateKey}";
- bool toolTableExists = TableExists(connection, toolTableName, commandTimeoutSeconds);
- bool deeplinkTableExists = TableExists(connection, deeplinkTableName, commandTimeoutSeconds);
- if (!toolTableExists && !deeplinkTableExists)
- {
- warnings.Add(new
- {
- date = reportDate.ToString("yyyy-MM-dd"),
- message = "两个来源日表都不存在,已跳过",
- toolTableName,
- deeplinkTableName
- });
- continue;
- }
- if (!allowPartialSources && (!toolTableExists || !deeplinkTableExists))
- {
- warnings.Add(new
- {
- date = reportDate.ToString("yyyy-MM-dd"),
- message = "来源日表不完整,已跳过;确认缺失表确实无数据后可传 allowPartialSources=true",
- toolTableName,
- toolTableExists,
- deeplinkTableName,
- deeplinkTableExists
- });
- continue;
- }
- try
- {
- var sourceCounts = new List<DeeplinkReportRepairSourceCount>();
- if (toolTableExists)
- {
- var toolCounts = SqlMapper.Query<DeeplinkReportToolSourceRow>(
- connection,
- $@"
- SELECT
- COALESCE(end_point, '') end_point,
- channel,
- COUNT(*) total_count,
- CAST(COALESCE(SUM(CASE WHEN success = 1 THEN 1 ELSE 0 END), 0) AS SIGNED) success_count
- FROM {toolTableName}
- GROUP BY COALESCE(end_point, ''), channel",
- commandTimeout: commandTimeoutSeconds
- );
- sourceCounts.AddRange(toolCounts.Select(item => new DeeplinkReportRepairSourceCount
- {
- source = toolTableName,
- source_endpoint = item.end_point?.Trim() ?? string.Empty,
- channel_name = GetDeeplinkReportChannelName(item.channel),
- total_count = item.total_count,
- success_count = item.success_count,
- fail_count = Math.Max(0, item.total_count - item.success_count)
- }));
- }
- if (deeplinkTableExists)
- {
- var deeplinkCounts = SqlMapper.Query<DeeplinkReportNamedSourceRow>(
- connection,
- $@"
- SELECT
- COALESCE(end_point, '') end_point,
- COALESCE(NULLIF(TRIM(channel_name), ''), 'unknown') channel_name,
- COUNT(*) total_count,
- CAST(COALESCE(SUM(CASE WHEN success = 1 THEN 1 ELSE 0 END), 0) AS SIGNED) success_count
- FROM {deeplinkTableName}
- GROUP BY COALESCE(end_point, ''), COALESCE(NULLIF(TRIM(channel_name), ''), 'unknown')",
- commandTimeout: commandTimeoutSeconds
- );
- sourceCounts.AddRange(deeplinkCounts.Select(item => new DeeplinkReportRepairSourceCount
- {
- source = deeplinkTableName,
- source_endpoint = item.end_point?.Trim() ?? string.Empty,
- channel_name = NormalizeDeeplinkReportChannelName(item.channel_name),
- total_count = item.total_count,
- success_count = item.success_count,
- fail_count = Math.Max(0, item.total_count - item.success_count)
- }));
- }
- var mergedCounts = new Dictionary<string, DeeplinkReportRepairCount>(StringComparer.OrdinalIgnoreCase);
- var remappedSources = new List<object>();
- foreach (var sourceCount in sourceCounts)
- {
- bool endpointMatched = endpointNames.TryGetValue(sourceCount.source_endpoint, out var endpointName);
- endpointName ??= fallbackNode.name;
- if (!endpointMatched)
- {
- remappedSources.Add(new
- {
- sourceCount.source,
- sourceEndpoint = sourceCount.source_endpoint,
- mappedEndpoint = endpointName,
- sourceCount.channel_name,
- sourceCount.total_count
- });
- }
- string mergedKey = $"{endpointName}\u001f{sourceCount.channel_name}";
- if (!mergedCounts.TryGetValue(mergedKey, out var mergedCount))
- {
- mergedCount = new DeeplinkReportRepairCount
- {
- endpoint_name = endpointName,
- channel_name = sourceCount.channel_name
- };
- mergedCounts[mergedKey] = mergedCount;
- }
- mergedCount.total_count += sourceCount.total_count;
- mergedCount.success_count += sourceCount.success_count;
- mergedCount.fail_count += sourceCount.fail_count;
- knownChannelNames.Add(sourceCount.channel_name);
- }
- var dateCounts = mergedCounts.Values
- .OrderBy(item => item.endpoint_name, StringComparer.OrdinalIgnoreCase)
- .ThenByDescending(item => item.total_count)
- .ThenBy(item => item.channel_name, StringComparer.OrdinalIgnoreCase)
- .ToList();
- var repairDate = new DeeplinkReportRepairDate
- {
- report_date = reportDate,
- tool_table_exists = toolTableExists,
- deeplink_table_exists = deeplinkTableExists,
- counts = dateCounts,
- remapped_sources = remappedSources
- };
- repairDates.Add(repairDate);
- }
- catch (Exception ex)
- {
- errors.Add(new
- {
- scope = "mysql_aggregate",
- date = reportDate.ToString("yyyy-MM-dd"),
- error = FormatRepairError(ex)
- });
- }
- }
- var redisReports = new List<object>();
- foreach (var endpoint in endpoints)
- {
- var endpointReports = new List<object>();
- string redisServer = EndPointCore.GetRedisServer(endpoint);
- try
- {
- await using var scope = await RedisClientFactory.CreateScopeAsync(redisServer);
- var redis = scope.Client;
- foreach (var repairDate in repairDates)
- {
- string dateKey = repairDate.report_date.ToString("yyyyMMdd");
- var endpointCounts = repairDate.counts
- .Where(item => item.endpoint_name.Equals(endpoint.name, StringComparison.OrdinalIgnoreCase))
- .ToList();
- var expectedParentCount = SumDeeplinkReportRepairCounts(endpointCounts);
- var beforeParentCount = await ReadDeeplinkReportBucketAsync(redis, "tool", dateKey);
- string channelIndexKey = $":parse_total:tool:channels:{dateKey}";
- var indexedChannelNames = await redis.SMembersAsync<string>(channelIndexKey) ?? [];
- var channelsToClear = new HashSet<string>(knownChannelNames, StringComparer.OrdinalIgnoreCase);
- channelsToClear.UnionWith(indexedChannelNames);
- channelsToClear.UnionWith(repairDate.counts
- .Select(item => item.channel_name)
- .Where(item => !string.IsNullOrWhiteSpace(item)));
- if (!dryRun)
- {
- await redis.DelAsync(channelIndexKey);
- foreach (var channelToClear in channelsToClear)
- {
- await DeleteDeeplinkReportBucketAsync(
- redis,
- $"tool:channel:{channelToClear}",
- dateKey
- );
- }
- await WriteDeeplinkReportBucketAsync(redis, "tool", dateKey, expectedParentCount);
- foreach (var endpointCount in endpointCounts)
- {
- await WriteDeeplinkReportBucketAsync(
- redis,
- $"tool:channel:{endpointCount.channel_name}",
- dateKey,
- endpointCount
- );
- }
- var endpointChannelNames = endpointCounts
- .Where(item => item.total_count > 0)
- .Select(item => item.channel_name)
- .Distinct(StringComparer.OrdinalIgnoreCase)
- .ToArray();
- if (endpointChannelNames.Length > 0)
- {
- await redis.SAddAsync(channelIndexKey, endpointChannelNames);
- await redis.ExpireAsync(channelIndexKey, DeeplinkReportStatsExpireSeconds);
- }
- }
- var afterParentCount = dryRun
- ? null
- : await ReadDeeplinkReportBucketAsync(redis, "tool", dateKey);
- endpointReports.Add(new
- {
- date = repairDate.report_date.ToString("yyyy-MM-dd"),
- before = beforeParentCount,
- expected = expectedParentCount,
- after = afterParentCount,
- channels = endpointCounts,
- clearedChannelCount = channelsToClear.Count
- });
- }
- }
- catch (Exception ex)
- {
- errors.Add(new
- {
- scope = "redis_write",
- endpoint = endpoint.name,
- error = FormatRepairError(ex)
- });
- }
- redisReports.Add(new
- {
- endpoint = endpoint.name,
- endpoint.description,
- reports = endpointReports
- });
- }
- var dailyReports = repairDates.Select(item => new
- {
- date = item.report_date.ToString("yyyy-MM-dd"),
- item.tool_table_exists,
- item.deeplink_table_exists,
- source = new
- {
- total_count = item.counts.Sum(count => count.total_count),
- success_count = item.counts.Sum(count => count.success_count),
- fail_count = item.counts.Sum(count => count.fail_count)
- },
- channels = item.counts
- .GroupBy(count => count.channel_name, StringComparer.OrdinalIgnoreCase)
- .Select(group => new
- {
- channel_name = group.Key,
- total_count = group.Sum(count => count.total_count),
- success_count = group.Sum(count => count.success_count),
- fail_count = group.Sum(count => count.fail_count)
- })
- .OrderByDescending(count => count.total_count)
- .ThenBy(count => count.channel_name, StringComparer.OrdinalIgnoreCase)
- .ToList(),
- endpoint_channels = item.counts,
- item.remapped_sources
- }).ToList();
- return new APIResult(new
- {
- success = errors.Count == 0,
- dryRun,
- allowPartialSources,
- allowCurrentDate,
- startDate = startDate.ToString("yyyy-MM-dd"),
- endDate = endDate.ToString("yyyy-MM-dd"),
- fallbackEndpoint = fallbackNode.name,
- sourceDatabase = new
- {
- endpoint = usesConfiguredParseDatabase ? parseAdminEndpoint!.name : EndPointCore.CurrentEndPoint,
- configured = usesConfiguredParseDatabase
- },
- processedDays = repairDates.Count,
- skippedDays = requestedDays - repairDates.Count,
- dailyReports,
- redis = redisReports,
- warnings,
- errors
- });
- }
- [HttpGet]
- public async Task<ActionResult> RepairTkDailyAccountStats(
- DateTime startDate = default,
- DateTime endDate = default,
- string accountIds = "129,140",
- string extraNames = "搜同款_楚颜_128众杰科技,搜同款_广哲2",
- bool dryRun = false,
- bool repairDailyLogs = true,
- bool repairRedis = true,
- bool repairRedisNameKeys = true,
- int commandTimeoutSeconds = 600)
- {
- if (startDate == default) startDate = new DateTime(2026, 6, 16);
- if (endDate == default) endDate = DateTime.Now.Date;
- commandTimeoutSeconds = Math.Clamp(commandTimeoutSeconds, 30, 3600);
- startDate = startDate.Date;
- endDate = endDate.Date;
- if (endDate < startDate)
- {
- return new APIResult(new { success = false, message = "endDate 不能早于 startDate" });
- }
- var ids = (accountIds ?? string.Empty)
- .Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)
- .Select(item => int.TryParse(item, out int id) ? id : 0)
- .Where(id => id > 0)
- .Distinct()
- .ToArray();
- if (ids.Length == 0)
- {
- return new APIResult(new { success = false, message = "accountIds 不能为空" });
- }
- using var conn = CenterHub.GetOpenConnection();
- if (conn.State != ConnectionState.Open) conn.Open();
- var errors = new List<object>();
- var affectedNames = new HashSet<string>(StringComparer.Ordinal);
- var accounts = SqlMapper.Query<TkDailyRepairCountRow>(
- conn,
- "SELECT id accountId, company accountName FROM tk_pool WHERE id IN @ids",
- new { ids },
- commandTimeout: commandTimeoutSeconds).ToList();
- foreach (string name in accounts.Select(item => item.accountName).Where(item => !string.IsNullOrWhiteSpace(item)))
- {
- affectedNames.Add(name);
- }
- extraNames ??= string.Empty;
- foreach (string name in extraNames
- .Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)
- .Where(item => !string.IsNullOrWhiteSpace(item)))
- {
- affectedNames.Add(name);
- }
- foreach (string name in SqlMapper.Query<string>(
- conn,
- @"
- SELECT DISTINCT accountName
- FROM center_daily_logs
- WHERE channel = 0
- AND accountId IN @ids
- AND log_date BETWEEN @startDate AND @endDate
- AND COALESCE(accountName,'')<>''",
- new { ids, startDate, endDate },
- commandTimeout: commandTimeoutSeconds))
- {
- affectedNames.Add(name);
- }
- var days = EachDay(startDate, endDate).ToList();
- var dailyReports = new List<object>();
- var endpointCountsByDate = new Dictionary<string, List<TkDailyRepairEndpointCountRow>>();
- foreach (var date in days)
- {
- try
- {
- string tableName = $"tk_parse_logs_{date:yyyyMMdd}";
- if (!TableExists(conn, tableName, commandTimeoutSeconds))
- {
- dailyReports.Add(new { date = date.ToString("yyyy-MM-dd"), tableName, skipped = true, reason = "table not exists" });
- continue;
- }
- var endpointCounts = SqlMapper.Query<TkDailyRepairEndpointCountRow>(
- conn,
- $@"
- SELECT
- l.accountId,
- COALESCE(p.company, MAX(l.accountName), '') accountName,
- COALESCE(l.end_point, '') end_point,
- COUNT(*) total_count,
- CAST(COALESCE(SUM(l.success = 1), 0) AS SIGNED) success_count,
- CAST(COALESCE(SUM(l.success = 0), 0) AS SIGNED) fail_count,
- CAST(COALESCE(SUM(l.message = '放弃转链' OR l.reason = '放弃转链'), 0) AS SIGNED) abandon_count
- FROM {tableName} l
- LEFT JOIN tk_pool p ON p.id = l.accountId
- WHERE l.accountId IN @ids
- GROUP BY l.accountId, p.company, l.end_point
- ORDER BY l.accountId, l.end_point",
- new { ids },
- commandTimeout: commandTimeoutSeconds).ToList();
- endpointCountsByDate[date.ToString("yyyyMMdd")] = endpointCounts;
- var accountCounts = endpointCounts
- .GroupBy(item => item.accountId)
- .Select(group => new TkDailyRepairCountRow
- {
- accountId = group.Key,
- accountName = accounts.FirstOrDefault(item => item.accountId == group.Key)?.accountName
- ?? group.FirstOrDefault()?.accountName
- ?? string.Empty,
- total_count = group.Sum(item => item.total_count),
- success_count = group.Sum(item => item.success_count),
- fail_count = group.Sum(item => item.fail_count),
- abandon_count = group.Sum(item => item.abandon_count)
- })
- .ToList();
- int dailyLogRows = 0;
- var allAccountCounts = ids
- .Select(accountId => accountCounts.FirstOrDefault(item => item.accountId == accountId)
- ?? new TkDailyRepairCountRow
- {
- accountId = accountId,
- accountName = accounts.FirstOrDefault(item => item.accountId == accountId)?.accountName ?? string.Empty
- })
- .ToList();
- foreach (var counts in allAccountCounts)
- {
- if (!dryRun && repairDailyLogs)
- {
- dailyLogRows += UpsertCenterDailyLog(conn, date, counts, commandTimeoutSeconds);
- }
- }
- dailyReports.Add(new
- {
- date = date.ToString("yyyy-MM-dd"),
- tableName,
- skipped = false,
- dailyLogRows,
- source = "mysql:tk_parse_logs_yyyyMMdd",
- counts = allAccountCounts,
- endpointCounts
- });
- }
- catch (Exception ex)
- {
- var error = new { scope = "daily_log", date = date.ToString("yyyy-MM-dd"), error = FormatRepairError(ex) };
- errors.Add(error);
- dailyReports.Add(new { date = date.ToString("yyyy-MM-dd"), skipped = true, reason = "error", error });
- }
- }
- var endpointReports = new List<object>();
- if (repairRedis || repairRedisNameKeys)
- {
- var endpoints = EndPointCore.List(true)
- .Where(node => node.status && node.is_public_api && !string.IsNullOrEmpty(EndPointCore.GetRedisServer(node)))
- .ToList();
- foreach (var endpoint in endpoints)
- {
- var redisServer = EndPointCore.GetRedisServer(endpoint);
- var endpointReport = new List<object>();
- try
- {
- await using var scope = await RedisClientFactory.CreateScopeAsync(redisServer);
- var redis = scope.Client;
- foreach (var date in days)
- {
- string dateKey = date.ToString("yyyyMMdd");
- if (!endpointCountsByDate.TryGetValue(dateKey, out var endpointCounts))
- {
- continue;
- }
- foreach (int accountId in ids)
- {
- var counts = endpointCounts.FirstOrDefault(item =>
- item.accountId == accountId &&
- string.Equals(item.end_point, endpoint.name, StringComparison.Ordinal))
- ?? new TkDailyRepairEndpointCountRow
- {
- accountId = accountId,
- accountName = accounts.FirstOrDefault(item => item.accountId == accountId)?.accountName ?? string.Empty,
- end_point = endpoint.name
- };
- if (!dryRun && repairRedis)
- {
- await WriteParseCountKeysAsync(redis, $"tb_{accountId}", dateKey, counts);
- }
- endpointReport.Add(new
- {
- date = date.ToString("yyyy-MM-dd"),
- bucket = $"tb_{accountId}",
- counts
- });
- }
- if (repairRedisNameKeys)
- {
- foreach (string name in affectedNames)
- {
- var before = await ReadParseCountKeysAsync(redis, name, dateKey);
- if (!dryRun)
- {
- await DeleteParseBucketKeysAsync(redis, name, dateKey);
- }
- endpointReport.Add(new
- {
- date = date.ToString("yyyy-MM-dd"),
- deletedBucket = name,
- before
- });
- }
- }
- }
- }
- catch (Exception ex)
- {
- errors.Add(new { scope = "redis_endpoint", endpoint = endpoint.name, error = FormatRepairError(ex) });
- }
- endpointReports.Add(new
- {
- endpoint = endpoint.name,
- endpoint.description,
- dryRun,
- deletedBuckets = endpointReport
- });
- }
- }
- return new APIResult(new
- {
- success = errors.Count == 0,
- dryRun,
- accountIds = ids,
- extraNames,
- repairDailyLogs,
- repairRedis,
- repairRedisNameKeys,
- commandTimeoutSeconds,
- startDate = startDate.ToString("yyyy-MM-dd"),
- endDate = endDate.ToString("yyyy-MM-dd"),
- affectedNames = affectedNames.OrderBy(item => item).ToList(),
- mysql = dailyReports,
- redis = endpointReports,
- errors
- });
- }
- /// <summary>
- /// 将淘宝覆盖规则命中产生的转链记录从 success=1 修正为 success=0,并同步修正各节点 Redis 统计。
- /// 覆盖规则在选择账号前命中,记录特征:channel=tb、linkType=unknown、accountId=0、itemId 为空。
- /// 按「日期 + 节点」处理:先修正该节点 Redis,成功后再更新对应 MySQL 记录;已修正的记录不会被重复计入。
- /// </summary>
- [HttpGet]
- public async Task<ActionResult> RepairTbOverrideSuccess(
- DateTime startTime = default,
- DateTime endTime = default,
- bool dryRun = true,
- int commandTimeoutSeconds = 600)
- {
- if (startTime == default) startTime = new DateTime(2026, 10, 20, 18, 52, 42);
- if (endTime == default) endTime = DateTime.Now;
- commandTimeoutSeconds = Math.Clamp(commandTimeoutSeconds, 30, 3600);
- if (endTime < startTime)
- {
- return new APIResult(new { success = false, message = "endTime 不能早于 startTime" });
- }
- const string condition = @"channel = @channel
- AND linkType = @linkType
- AND success = 1
- AND accountId = 0
- AND COALESCE(itemId,'') = ''
- AND create_time BETWEEN @from AND @to";
- using var conn = CenterHub.GetOpenConnection();
- if (conn.State != ConnectionState.Open) conn.Open();
- // 与 TkLogCore.GetTotalAsync 读取统计时的节点范围保持一致
- var endpoints = EndPointCore.List()
- .Where(node => node.is_public_api && !string.IsNullOrEmpty(EndPointCore.GetRedisServer(node)))
- .GroupBy(node => node.name, StringComparer.Ordinal)
- .ToDictionary(group => group.Key, group => group.First(), StringComparer.Ordinal);
- var errors = new List<object>();
- var dayReports = new List<object>();
- foreach (var date in EachDay(startTime.Date, endTime.Date))
- {
- DateTime from = date < startTime ? startTime : date;
- DateTime dayEnd = date.AddDays(1).AddTicks(-1);
- DateTime to = dayEnd > endTime ? endTime : dayEnd;
- string tableName = $"tk_parse_logs_{date:yyyyMMdd}";
- try
- {
- if (!TableExists(conn, tableName, commandTimeoutSeconds))
- {
- dayReports.Add(new { date = date.ToString("yyyy-MM-dd"), tableName, skipped = true, reason = "table not exists" });
- continue;
- }
- var args = new { channel = (int)TkChannelEnum.tb, linkType = (int)LinkTypeEnum.unknown, from, to };
- var groups = SqlMapper.Query<TbOverrideRepairGroupRow>(
- conn,
- $@"
- SELECT
- COALESCE(end_point,'') end_point,
- DATE_FORMAT(create_time,'%Y%m%d%H') hour_key,
- COALESCE(deeplink_url,'') deeplink_url,
- COALESCE(riskStrategy,'') riskStrategy,
- launchScene,
- COUNT(*) row_count
- FROM {tableName}
- WHERE {condition}
- GROUP BY COALESCE(end_point,''), DATE_FORMAT(create_time,'%Y%m%d%H'), COALESCE(deeplink_url,''), COALESCE(riskStrategy,''), launchScene",
- args,
- commandTimeout: commandTimeoutSeconds).ToList();
- var summary = SqlMapper.Query<TbOverrideRepairSummaryRow>(
- conn,
- $@"
- SELECT CAST(subCode AS CHAR) subCode, COALESCE(message,'') message, COALESCE(reason,'') reason, COUNT(*) row_count
- FROM {tableName}
- WHERE {condition}
- GROUP BY CAST(subCode AS CHAR), COALESCE(message,''), COALESCE(reason,'')
- ORDER BY row_count DESC",
- args,
- commandTimeout: commandTimeoutSeconds).ToList();
- var topDeeplinks = groups
- .GroupBy(item => item.deeplink_url)
- .Select(group => new { deeplink_url = group.Key, row_count = group.Sum(item => item.row_count) })
- .OrderByDescending(item => item.row_count)
- .Take(20)
- .ToList();
- var endpointReports = new List<object>();
- foreach (var endpointGroup in groups.GroupBy(item => item.end_point))
- {
- long rowCount = endpointGroup.Sum(item => item.row_count);
- if (!endpoints.TryGetValue(endpointGroup.Key, out var endpoint))
- {
- errors.Add(new { scope = "endpoint_not_found", date = date.ToString("yyyy-MM-dd"), end_point = endpointGroup.Key, rowCount });
- continue;
- }
- var deltas = BuildTbOverrideRepairDeltas(endpointGroup);
- var keyReports = new List<object>();
- int appliedKeys = 0;
- Dictionary<string, int> mysql;
- try
- {
- await using var scope = await RedisClientFactory.CreateScopeAsync(EndPointCore.GetRedisServer(endpoint));
- var redis = scope.Client;
- foreach (var (key, (delta, expireSeconds)) in deltas)
- {
- long before = await redis.GetAsync<long>(key);
- // 只减到 0 为止,避免计数出现负数
- long applied = delta < 0 ? -Math.Min(-delta, Math.Max(before, 0)) : delta;
- if (!dryRun && applied != 0)
- {
- await redis.IncrByAsync(key, applied);
- if (before == 0 && applied > 0 && expireSeconds > 0)
- {
- await redis.ExpireAsync(key, expireSeconds);
- }
- appliedKeys++;
- }
- // 小时级 key 数量较多,只在报告中列出日、月级 key
- if (!IsHourlyStatsKey(key))
- {
- keyReports.Add(new { key, before, delta, applied });
- }
- }
- }
- catch (Exception ex)
- {
- errors.Add(new
- {
- scope = "redis_endpoint",
- date = date.ToString("yyyy-MM-dd"),
- end_point = endpoint.name,
- appliedKeys,
- totalKeys = deltas.Count,
- error = FormatRepairError(ex)
- });
- continue;
- }
- try
- {
- mysql = RepairTbOverrideMysqlRows(conn, date, condition, args, endpoint.name, dryRun, commandTimeoutSeconds);
- }
- catch (Exception ex)
- {
- errors.Add(new { scope = "mysql_endpoint", date = date.ToString("yyyy-MM-dd"), end_point = endpoint.name, redisApplied = !dryRun, error = FormatRepairError(ex) });
- continue;
- }
- endpointReports.Add(new
- {
- end_point = endpoint.name,
- rowCount,
- totalKeys = deltas.Count,
- appliedKeys,
- mysql,
- keys = keyReports
- });
- }
- dayReports.Add(new
- {
- date = date.ToString("yyyy-MM-dd"),
- from,
- to,
- rowCount = groups.Sum(item => item.row_count),
- summary,
- topDeeplinks,
- endpoints = endpointReports
- });
- }
- catch (Exception ex)
- {
- errors.Add(new { scope = "day", date = date.ToString("yyyy-MM-dd"), tableName, error = FormatRepairError(ex) });
- }
- }
- return new APIResult(new
- {
- success = errors.Count == 0,
- dryRun,
- startTime,
- endTime,
- days = dayReports,
- errors
- });
- }
- /// <summary>
- /// 计算单个节点需要修正的 Redis 计数增量,对应 TkLogCore.ParseLogAsync 中 success 相关的写入:
- /// parse_total 的 total/success/fail(dp_{flag} 可能由 success 变为 fail)以及 parse_metric 的 success。
- /// </summary>
- private static Dictionary<string, (long delta, int expireSeconds)> BuildTbOverrideRepairDeltas(IEnumerable<TbOverrideRepairGroupRow> rows)
- {
- const string channel = "tb";
- var deltas = new Dictionary<string, (long delta, int expireSeconds)>(StringComparer.Ordinal);
- void Add(string key, long delta, int expireSeconds)
- {
- deltas.TryGetValue(key, out var current);
- deltas[key] = (current.delta + delta, expireSeconds);
- }
- foreach (var row in rows)
- {
- var metricTime = DateTime.ParseExact(row.hour_key, "yyyyMMddHH", null);
- string[] timeKeys = [metricTime.ToString("yyyyMM"), metricTime.ToString("yyyyMMdd"), row.hour_key];
- string oldFlag = TkLogCore.GetParseDeeplinkFlag(row.deeplink_url, true);
- string newFlag = TkLogCore.GetParseDeeplinkFlag(row.deeplink_url, false);
- string[] oldBuckets = [$"dp_{oldFlag}:all", $"dp_{oldFlag}:{channel}", "all", $"dp_channel:{channel}", channel];
- string[] newBuckets = [$"dp_{newFlag}:all", $"dp_{newFlag}:{channel}", "all", $"dp_channel:{channel}", channel];
- foreach (string timeKey in timeKeys)
- {
- int expireSeconds = TkLogCore.GetStatsExpireSeconds(timeKey);
- foreach (string bucket in oldBuckets)
- {
- Add($":parse_total:{bucket}:{timeKey}", -row.row_count, expireSeconds);
- Add($":parse_total:{bucket}:success:{timeKey}", -row.row_count, expireSeconds);
- }
- foreach (string bucket in newBuckets)
- {
- Add($":parse_total:{bucket}:{timeKey}", row.row_count, expireSeconds);
- Add($":parse_total:{bucket}:fail:{timeKey}", row.row_count, expireSeconds);
- }
- }
- foreach (string key in TracksCore.BuildParseMetricSuccessKeys(metricTime, channel, row.riskStrategy, row.launchScene))
- {
- Add(key, -row.row_count, 0);
- }
- }
- return deltas
- .Where(item => item.Value.delta != 0)
- .ToDictionary(item => item.Key, item => item.Value, StringComparer.Ordinal);
- }
- private static bool IsHourlyStatsKey(string key)
- {
- int index = key.LastIndexOf(':');
- string last = index >= 0 ? key[(index + 1)..] : key;
- if (last.Length == 10 && last.All(char.IsDigit)) return true;
- // parse_metric key 的时间段位于中间::parse_metric:hour:success:{yyyyMMddHH}:...
- return key.Contains(":parse_metric:hour:", StringComparison.Ordinal);
- }
- /// <summary>
- /// 修正(dryRun 时仅统计)单个节点在指定日期的覆盖规则记录:
- /// 日表、dp 日表、测试表改为 success=0;success 汇总表删除对应记录。
- /// </summary>
- private static Dictionary<string, int> RepairTbOverrideMysqlRows(
- IDbConnection conn,
- DateTime date,
- string condition,
- object args,
- string endPoint,
- bool dryRun,
- int commandTimeoutSeconds)
- {
- var parameters = new DynamicParameters(args);
- parameters.Add("end_point", endPoint);
- string where = $"{condition} AND COALESCE(end_point,'') = @end_point";
- string dpWhere = $"{where} AND parse_type = 'dp'";
- var targets = new (string table, string where, bool delete)[]
- {
- ($"tk_parse_logs_{date:yyyyMMdd}", where, false),
- ($"tk_dp_logs_{date:yyyyMMdd}", dpWhere, false),
- ("tk_parse_logs_test", where, false),
- ("tk_success_parse_logs", where, true),
- ("tk_dp_logs_success", dpWhere, true),
- };
- var result = new Dictionary<string, int>(StringComparer.Ordinal);
- foreach (var (table, tableWhere, delete) in targets)
- {
- if (!TableExists(conn, table, commandTimeoutSeconds)) continue;
- string sql = dryRun
- ? $"SELECT COUNT(*) FROM {table} WHERE {tableWhere}"
- : delete
- ? $"DELETE FROM {table} WHERE {tableWhere}"
- : $"UPDATE {table} SET success = 0 WHERE {tableWhere}";
- result[table] = dryRun
- ? SqlMapper.ExecuteScalar<int>(conn, sql, parameters, commandTimeout: commandTimeoutSeconds)
- : SqlMapper.Execute(conn, sql, parameters, commandTimeout: commandTimeoutSeconds);
- }
- return result;
- }
- private sealed class TbOverrideRepairGroupRow
- {
- public string end_point { get; set; } = string.Empty;
- public string hour_key { get; set; } = string.Empty;
- public string deeplink_url { get; set; } = string.Empty;
- public string riskStrategy { get; set; } = string.Empty;
- public int launchScene { get; set; }
- public long row_count { get; set; }
- }
- private sealed class TbOverrideRepairSummaryRow
- {
- public string subCode { get; set; } = string.Empty;
- public string message { get; set; } = string.Empty;
- public string reason { get; set; } = string.Empty;
- public long row_count { get; set; }
- }
- private static HashSet<string> GetKnownDeeplinkReportChannelNames()
- {
- var channelNames = Enum.GetNames(typeof(TkChannelEnum))
- .ToHashSet(StringComparer.OrdinalIgnoreCase);
- try
- {
- var rules = new DBContext.Table("deeplink_parse_rule")
- .Select<DeeplinkParseRuleDTO>() ?? Enumerable.Empty<DeeplinkParseRuleDTO>();
- foreach (var rule in rules)
- {
- if (!string.IsNullOrWhiteSpace(rule.channel_name))
- {
- channelNames.Add(rule.channel_name.Trim());
- }
- }
- }
- catch
- {
- // The source log rows and Redis channel index still provide enough information.
- }
- return channelNames;
- }
- private static async Task<ActionResult> RepairDeeplinkReportStatsFromLegacyRedisAsync(
- List<EndPointDTO> endpoints,
- HashSet<string> knownChannelNames,
- DateTime startDate,
- DateTime endDate,
- bool dryRun)
- {
- var sourceChannelNames = knownChannelNames
- .Where(IsLegacyDeeplinkReportSourceChannel)
- .OrderBy(channelName => channelName, StringComparer.OrdinalIgnoreCase)
- .ToList();
- var requestedDates = EachDay(startDate, endDate).ToList();
- var countsByDate = requestedDates.ToDictionary(
- reportDate => reportDate,
- _ => new List<DeeplinkReportRepairCount>()
- );
- var endpointReports = new List<object>();
- var warnings = new List<object>();
- var errors = new List<object>();
- foreach (var endpoint in endpoints)
- {
- var dateReports = new List<object>();
- string redisServer = EndPointCore.GetRedisServer(endpoint);
- try
- {
- await using var scope = await RedisClientFactory.CreateScopeAsync(redisServer);
- var redis = scope.Client;
- foreach (var reportDate in requestedDates)
- {
- string dateKey = reportDate.ToString("yyyyMMdd");
- var readTasks = sourceChannelNames.Select(async channelName =>
- {
- var legacyCount = await ReadDeeplinkReportBucketAsync(redis, channelName, dateKey);
- legacyCount.endpoint_name = endpoint.name;
- legacyCount.channel_name = channelName;
- return legacyCount;
- });
- var legacyCounts = (await Task.WhenAll(readTasks))
- .Where(count => count.total_count > 0 || count.success_count > 0 || count.fail_count > 0)
- .OrderByDescending(count => count.total_count)
- .ThenBy(count => count.channel_name, StringComparer.OrdinalIgnoreCase)
- .ToList();
- var expectedParentCount = SumDeeplinkReportRepairCounts(legacyCounts);
- var beforeParentCount = await ReadDeeplinkReportBucketAsync(redis, "tool", dateKey);
- if (expectedParentCount.total_count <= 0)
- {
- dateReports.Add(new
- {
- date = reportDate.ToString("yyyy-MM-dd"),
- sourceFound = false,
- before = beforeParentCount,
- message = "未找到旧渠道统计键,未执行清理或覆盖"
- });
- continue;
- }
- countsByDate[reportDate].AddRange(legacyCounts);
- string channelIndexKey = $":parse_total:tool:channels:{dateKey}";
- var indexedChannelNames = await redis.SMembersAsync<string>(channelIndexKey) ?? [];
- var channelsToClear = new HashSet<string>(sourceChannelNames, StringComparer.OrdinalIgnoreCase);
- channelsToClear.UnionWith(indexedChannelNames);
- if (!dryRun)
- {
- await redis.DelAsync(channelIndexKey);
- foreach (var channelToClear in channelsToClear)
- {
- await DeleteDeeplinkReportBucketAsync(
- redis,
- $"tool:channel:{channelToClear}",
- dateKey
- );
- }
- await WriteDeeplinkReportBucketAsync(redis, "tool", dateKey, expectedParentCount);
- foreach (var legacyCount in legacyCounts)
- {
- await WriteDeeplinkReportBucketAsync(
- redis,
- $"tool:channel:{legacyCount.channel_name}",
- dateKey,
- legacyCount
- );
- }
- string[] populatedChannelNames = legacyCounts
- .Where(count => count.total_count > 0)
- .Select(count => count.channel_name)
- .Distinct(StringComparer.OrdinalIgnoreCase)
- .ToArray();
- if (populatedChannelNames.Length > 0)
- {
- await redis.SAddAsync(channelIndexKey, populatedChannelNames);
- await redis.ExpireAsync(channelIndexKey, DeeplinkReportStatsExpireSeconds);
- }
- }
- var afterParentCount = dryRun
- ? null
- : await ReadDeeplinkReportBucketAsync(redis, "tool", dateKey);
- dateReports.Add(new
- {
- date = reportDate.ToString("yyyy-MM-dd"),
- sourceFound = true,
- before = beforeParentCount,
- expected = expectedParentCount,
- after = afterParentCount,
- channels = legacyCounts,
- sourceKeysPreserved = true
- });
- }
- }
- catch (Exception ex)
- {
- errors.Add(new
- {
- scope = "legacy_redis",
- endpoint = endpoint.name,
- error = FormatRepairError(ex)
- });
- }
- endpointReports.Add(new
- {
- endpoint = endpoint.name,
- endpoint.description,
- reports = dateReports
- });
- }
- foreach (var reportDate in requestedDates.Where(date => countsByDate[date].Count == 0))
- {
- warnings.Add(new
- {
- date = reportDate.ToString("yyyy-MM-dd"),
- message = "所有节点均未找到旧渠道统计,不会修改该日数据"
- });
- }
- var processedDates = requestedDates
- .Where(date => countsByDate[date].Count > 0)
- .ToList();
- var dailyReports = processedDates.Select(reportDate =>
- {
- var endpointChannelCounts = countsByDate[reportDate];
- var channelCounts = endpointChannelCounts
- .GroupBy(count => count.channel_name, StringComparer.OrdinalIgnoreCase)
- .Select(group => new
- {
- channel_name = group.Key,
- total_count = group.Sum(count => count.total_count),
- success_count = group.Sum(count => count.success_count),
- fail_count = group.Sum(count => count.fail_count)
- })
- .OrderByDescending(count => count.total_count)
- .ThenBy(count => count.channel_name, StringComparer.OrdinalIgnoreCase)
- .ToList();
- return new
- {
- date = reportDate.ToString("yyyy-MM-dd"),
- total_count = channelCounts.Sum(count => count.total_count),
- success_count = channelCounts.Sum(count => count.success_count),
- fail_count = channelCounts.Sum(count => count.fail_count),
- channels = channelCounts,
- endpoint_channels = endpointChannelCounts
- };
- }).ToList();
- return new APIResult(new
- {
- success = errors.Count == 0,
- dryRun,
- useLegacyRedis = true,
- sourceMode = "legacyRedis",
- sourceKeysPreserved = true,
- parentTotalRule = "sum(channel totals)",
- startDate = startDate.ToString("yyyy-MM-dd"),
- endDate = endDate.ToString("yyyy-MM-dd"),
- processedDays = processedDates.Count,
- skippedDays = requestedDates.Count - processedDates.Count,
- sourceChannels = sourceChannelNames,
- dailyReports,
- redis = endpointReports,
- warnings,
- errors
- });
- }
- private static bool IsLegacyDeeplinkReportSourceChannel(string channelName)
- {
- if (string.IsNullOrWhiteSpace(channelName)) return false;
- return !channelName.Equals("all", StringComparison.OrdinalIgnoreCase) &&
- !channelName.Equals("tool", StringComparison.OrdinalIgnoreCase) &&
- !channelName.Equals("unknown", StringComparison.OrdinalIgnoreCase) &&
- !channelName.Equals("legacy_unclassified", StringComparison.OrdinalIgnoreCase);
- }
- private static string GetDeeplinkReportChannelName(int channelValue)
- {
- if (Enum.IsDefined(typeof(TkChannelEnum), channelValue))
- {
- return ((TkChannelEnum)channelValue).ToString();
- }
- return $"channel_{channelValue}";
- }
- private static string NormalizeDeeplinkReportChannelName(string channelName)
- {
- return string.IsNullOrWhiteSpace(channelName) ? "unknown" : channelName.Trim();
- }
- private static DeeplinkReportRepairCount SumDeeplinkReportRepairCounts(
- IEnumerable<DeeplinkReportRepairCount> counts)
- {
- return new DeeplinkReportRepairCount
- {
- total_count = counts.Sum(item => item.total_count),
- success_count = counts.Sum(item => item.success_count),
- fail_count = counts.Sum(item => item.fail_count)
- };
- }
- private static async Task<DeeplinkReportRepairCount> ReadDeeplinkReportBucketAsync(
- YunhuiKit.RedisClient redis,
- string bucket,
- string dateKey)
- {
- return new DeeplinkReportRepairCount
- {
- total_count = await redis.GetAsync<long>($":parse_total:{bucket}:{dateKey}"),
- success_count = await redis.GetAsync<long>($":parse_total:{bucket}:success:{dateKey}"),
- fail_count = await redis.GetAsync<long>($":parse_total:{bucket}:fail:{dateKey}")
- };
- }
- private static Task<long> DeleteDeeplinkReportBucketAsync(
- YunhuiKit.RedisClient redis,
- string bucket,
- string dateKey)
- {
- return redis.DelAsync(
- $":parse_total:{bucket}:{dateKey}",
- $":parse_total:{bucket}:success:{dateKey}",
- $":parse_total:{bucket}:fail:{dateKey}"
- );
- }
- private static async Task WriteDeeplinkReportBucketAsync(
- YunhuiKit.RedisClient redis,
- string bucket,
- string dateKey,
- DeeplinkReportRepairCount counts)
- {
- if (counts.total_count <= 0)
- {
- await DeleteDeeplinkReportBucketAsync(redis, bucket, dateKey);
- return;
- }
- bool totalWritten = await redis.SetAsync(
- $":parse_total:{bucket}:{dateKey}",
- counts.total_count,
- DeeplinkReportStatsExpireSeconds
- );
- bool successWritten = await redis.SetAsync(
- $":parse_total:{bucket}:success:{dateKey}",
- counts.success_count,
- DeeplinkReportStatsExpireSeconds
- );
- bool failWritten = await redis.SetAsync(
- $":parse_total:{bucket}:fail:{dateKey}",
- counts.fail_count,
- DeeplinkReportStatsExpireSeconds
- );
- if (!totalWritten || !successWritten || !failWritten)
- {
- throw new InvalidOperationException($"Redis 写入失败:bucket={bucket}, date={dateKey}");
- }
- }
- private sealed class DeeplinkReportToolSourceRow
- {
- public string end_point { get; set; } = string.Empty;
- public int channel { get; set; }
- public long total_count { get; set; }
- public long success_count { get; set; }
- }
- private sealed class DeeplinkReportNamedSourceRow
- {
- public string end_point { get; set; } = string.Empty;
- public string channel_name { get; set; } = string.Empty;
- public long total_count { get; set; }
- public long success_count { get; set; }
- }
- private sealed class DeeplinkReportRepairSourceCount
- {
- public string source { get; set; } = string.Empty;
- public string source_endpoint { get; set; } = string.Empty;
- public string channel_name { get; set; } = string.Empty;
- public long total_count { get; set; }
- public long success_count { get; set; }
- public long fail_count { get; set; }
- }
- private sealed class DeeplinkReportRepairCount
- {
- public string endpoint_name { get; set; } = string.Empty;
- public string channel_name { get; set; } = string.Empty;
- public long total_count { get; set; }
- public long success_count { get; set; }
- public long fail_count { get; set; }
- }
- private sealed class DeeplinkReportRepairDate
- {
- public DateTime report_date { get; set; }
- public bool tool_table_exists { get; set; }
- public bool deeplink_table_exists { get; set; }
- public List<DeeplinkReportRepairCount> counts { get; set; } = [];
- public List<object> remapped_sources { get; set; } = [];
- }
- private static IEnumerable<DateTime> EachDay(DateTime startDate, DateTime endDate)
- {
- for (var date = startDate.Date; date <= endDate.Date; date = date.AddDays(1))
- {
- yield return date;
- }
- }
- private static bool TableExists(IDbConnection conn, string tableName, int commandTimeoutSeconds)
- {
- const string sql = @"
- SELECT COUNT(*)
- FROM information_schema.tables
- WHERE table_schema = DATABASE()
- AND table_name = @tableName";
- return SqlMapper.ExecuteScalar<int>(
- conn,
- sql,
- new { tableName },
- commandTimeout: commandTimeoutSeconds) > 0;
- }
- private static int UpsertCenterDailyLog(IDbConnection conn, DateTime date, TkDailyRepairCountRow counts, int commandTimeoutSeconds)
- {
- if (string.IsNullOrWhiteSpace(counts.accountName)) return 0;
- string successPercentage = counts.total_count > 0 ? $"{counts.success_count / (double)counts.total_count * 100:f2}%" : string.Empty;
- string abandonPercentage = counts.total_count > 0 ? $"{counts.abandon_count / (double)counts.total_count * 100:f2}%" : string.Empty;
- var existingIds = SqlMapper.Query<int>(
- conn,
- @"
- SELECT id
- FROM center_daily_logs
- WHERE channel = 0
- AND log_date = @date
- AND accountId = @accountId
- ORDER BY id",
- new { date, counts.accountId },
- commandTimeout: commandTimeoutSeconds).ToList();
- if (existingIds.Count > 0)
- {
- int affectedRows = SqlMapper.Execute(
- conn,
- @"
- UPDATE center_daily_logs
- SET accountName = @accountName,
- parse_total_count = @totalCount,
- parse_success_count = @successCount,
- parse_abandon_count = @abandonCount,
- parse_success_percentage = @successPercentage,
- parse_abandon_percentage = @abandonPercentage,
- last_time = NOW()
- WHERE id = @id",
- new
- {
- id = existingIds[0],
- counts.accountName,
- totalCount = counts.total_count,
- successCount = counts.success_count,
- abandonCount = counts.abandon_count,
- successPercentage,
- abandonPercentage
- },
- commandTimeout: commandTimeoutSeconds);
- if (existingIds.Count > 1)
- {
- affectedRows += SqlMapper.Execute(
- conn,
- "DELETE FROM center_daily_logs WHERE id IN @ids",
- new { ids = existingIds.Skip(1).ToArray() },
- commandTimeout: commandTimeoutSeconds);
- }
- return affectedRows;
- }
- if (counts.total_count <= 0) return 0;
- return SqlMapper.Execute(
- conn,
- @"
- INSERT INTO center_daily_logs
- (channel, accountId, accountName, log_date, create_time, last_time,
- parse_total_count, parse_success_count, parse_abandon_count,
- parse_success_percentage, parse_abandon_percentage)
- VALUES
- (0, @accountId, @accountName, @date, NOW(), NOW(),
- @totalCount, @successCount, @abandonCount,
- @successPercentage, @abandonPercentage)",
- new
- {
- date,
- counts.accountId,
- counts.accountName,
- totalCount = counts.total_count,
- successCount = counts.success_count,
- abandonCount = counts.abandon_count,
- successPercentage,
- abandonPercentage
- },
- commandTimeout: commandTimeoutSeconds);
- }
- private static async Task<TkDailyRepairCountRow> ReadParseCountKeysAsync(YunhuiKit.RedisClient redis, string bucket, string dateKey)
- {
- return new TkDailyRepairCountRow
- {
- accountName = bucket,
- total_count = await redis.GetAsync<int>($":parse_total:{bucket}:{dateKey}"),
- success_count = await redis.GetAsync<int>($":parse_total:{bucket}:success:{dateKey}"),
- fail_count = await redis.GetAsync<int>($":parse_total:{bucket}:fail:{dateKey}"),
- abandon_count = await redis.GetAsync<int>($":parse_total:{bucket}:放弃转链:{dateKey}")
- };
- }
- private static async Task DeleteParseBucketKeysAsync(YunhuiKit.RedisClient redis, string bucket, string dateKey)
- {
- var keys = new List<string>
- {
- $":parse_total:{bucket}:{dateKey}",
- $":parse_total:{bucket}:success:{dateKey}",
- $":parse_total:{bucket}:fail:{dateKey}",
- $":parse_total:{bucket}:放弃转链:{dateKey}",
- $":parse_total:{bucket}:message:{dateKey}",
- $":parse_total:{bucket}:reason:{dateKey}"
- };
- foreach (string dpBucket in new[] { "dp_none", "dp_home", "dp_success", "dp_fail" })
- {
- keys.Add($":parse_total:{dpBucket}:{bucket}:{dateKey}");
- keys.Add($":parse_total:{dpBucket}:{bucket}:success:{dateKey}");
- keys.Add($":parse_total:{dpBucket}:{bucket}:fail:{dateKey}");
- keys.Add($":parse_total:{dpBucket}:{bucket}:放弃转链:{dateKey}");
- keys.Add($":parse_total:{dpBucket}:{bucket}:message:{dateKey}");
- keys.Add($":parse_total:{dpBucket}:{bucket}:reason:{dateKey}");
- }
- await redis.DelAsync(keys.ToArray());
- }
- private static string FormatRepairError(Exception ex)
- {
- return ex.InnerException == null ? ex.Message : $"{ex.Message} | {ex.InnerException.Message}";
- }
- private static async Task WriteParseCountKeysAsync(YunhuiKit.RedisClient redis, string bucket, string dateKey, TkDailyRepairCountRow counts)
- {
- var keys = new[]
- {
- $":parse_total:{bucket}:{dateKey}",
- $":parse_total:{bucket}:success:{dateKey}",
- $":parse_total:{bucket}:fail:{dateKey}",
- $":parse_total:{bucket}:放弃转链:{dateKey}"
- };
- if (counts.total_count <= 0)
- {
- await redis.DelAsync(keys);
- return;
- }
- await redis.SetAsync(keys[0], counts.total_count, 90 * 86400);
- await redis.SetAsync(keys[1], counts.success_count, 90 * 86400);
- await redis.SetAsync(keys[2], counts.fail_count, 90 * 86400);
- await redis.SetAsync(keys[3], counts.abandon_count, 90 * 86400);
- }
- private class TkDailyRepairCountRow
- {
- public int accountId { get; set; }
- public string accountName { get; set; } = string.Empty;
- public long total_count { get; set; }
- public long success_count { get; set; }
- public long fail_count { get; set; }
- public long abandon_count { get; set; }
- }
- private sealed class TkDailyRepairEndpointCountRow : TkDailyRepairCountRow
- {
- public string end_point { get; set; } = string.Empty;
- }
- private static async Task<string> SaveJdStressStartLogAsync(string runId, string logName, int t, int time, string ip, string oaid, bool wait)
- {
- try
- {
- await new LoggerLibrary("jd_api_stress", logName)
- .AppendLine($"status=started")
- .AppendLine($"runId={runId}")
- .AppendLine($"startedAt={DateTime.Now:O}")
- .AppendLine($"t={t}")
- .AppendLine($"timeSeconds={time}")
- .AppendLine($"ip={ip}")
- .AppendLine($"oaid={oaid}")
- .AppendLine($"wait={wait}")
- .AppendLine($"content={JdStressDeeplinkContent}")
- .SaveAsync();
- return "started";
- }
- catch (Exception ex)
- {
- return ex.Message;
- }
- }
- private static async Task<JdStressRunResult> RunJdStressAsync(string runId, string logName, int t, int time, string ip, string oaid, string startLogStatus, CancellationToken cancellationToken)
- {
- var startedAt = DateTime.Now;
- var stopwatch = Stopwatch.StartNew();
- long scheduledCount = 0;
- long completedCount = 0;
- long successCount = 0;
- long failCount = 0;
- long exceptionCount = 0;
- long riskControlCount = 0;
- long totalElapsedMs = 0;
- int activeCount = 0;
- int peakActiveCount = 0;
- var responseLines = new ConcurrentQueue<string>();
- var messageCounts = new ConcurrentDictionary<string, int>(StringComparer.Ordinal);
- var codeCounts = new ConcurrentDictionary<string, int>(StringComparer.Ordinal);
- int initialTaskCapacity = (int)Math.Min((long)t * Math.Min(time, 60), 4096L);
- var runningTasks = new List<Task>(initialTaskCapacity);
- string message = "ok";
- try
- {
- for (int second = 0; second < time && !cancellationToken.IsCancellationRequested; second++)
- {
- for (int i = 0; i < t; i++)
- {
- long sequence = Interlocked.Increment(ref scheduledCount);
- runningTasks.Add(Task.Run(async () =>
- {
- int currentActive = Interlocked.Increment(ref activeCount);
- UpdateMax(ref peakActiveCount, currentActive);
- try
- {
- var item = await ExecuteJdStressRequestAsync(sequence, ip, oaid, cancellationToken);
- Interlocked.Increment(ref completedCount);
- Interlocked.Add(ref totalElapsedMs, item.ElapsedMs);
- if (item.Success)
- {
- Interlocked.Increment(ref successCount);
- }
- else
- {
- Interlocked.Increment(ref failCount);
- }
- if (item.IsException)
- {
- Interlocked.Increment(ref exceptionCount);
- }
- if (IsRiskControlResult(item))
- {
- Interlocked.Increment(ref riskControlCount);
- }
- string messageKey = string.IsNullOrWhiteSpace(item.SubMessage)
- ? item.Message
- : $"{item.Message}:{item.SubMessage}";
- if (string.IsNullOrWhiteSpace(messageKey)) messageKey = "empty";
- messageCounts.AddOrUpdate(messageKey, 1, (_, value) => value + 1);
- codeCounts.AddOrUpdate(item.Code.ToString(), 1, (_, value) => value + 1);
- responseLines.Enqueue(
- $"{item.FinishedAt:O}\tseq={item.Sequence}\telapsedMs={item.ElapsedMs}\tcode={item.Code}\tsuccess={item.Success}\tmessage={NormalizeLogValue(item.Message)}\tsub_message={NormalizeLogValue(item.SubMessage)}\tresponse={NormalizeLogValue(item.Response)}");
- }
- finally
- {
- Interlocked.Decrement(ref activeCount);
- }
- }));
- }
- await ObserveCompletedTasksAsync(runningTasks);
- var nextTick = TimeSpan.FromSeconds(second + 1);
- var delay = nextTick - stopwatch.Elapsed;
- if (delay > TimeSpan.Zero)
- {
- await Task.Delay(delay, cancellationToken);
- }
- }
- }
- catch (OperationCanceledException ex)
- {
- message = $"canceled:{ex.Message}";
- responseLines.Enqueue($"{DateTime.Now:O}\tmessage={NormalizeLogValue(message)}");
- }
- catch (Exception ex)
- {
- message = $"runner error:{ex.Message}";
- responseLines.Enqueue($"{DateTime.Now:O}\tmessage={NormalizeLogValue(message)}\tstack={NormalizeLogValue(ex.StackTrace ?? string.Empty)}");
- }
- try
- {
- await Task.WhenAll(runningTasks);
- }
- catch (Exception ex)
- {
- message = $"task wait error:{ex.Message}";
- responseLines.Enqueue($"{DateTime.Now:O}\tmessage={NormalizeLogValue(message)}\tstack={NormalizeLogValue(ex.StackTrace ?? string.Empty)}");
- }
- stopwatch.Stop();
- string logStatus = "saved";
- try
- {
- var log = new LoggerLibrary("jd_api_stress", logName);
- log.AppendLine($"status=finished");
- log.AppendLine($"startLogStatus={startLogStatus}");
- log.AppendLine($"runId={runId}");
- log.AppendLine($"startedAt={startedAt:O}");
- log.AppendLine($"finishedAt={DateTime.Now:O}");
- log.AppendLine($"t={t}");
- log.AppendLine($"timeSeconds={time}");
- log.AppendLine($"ip={ip}");
- log.AppendLine($"oaid={oaid}");
- log.AppendLine($"content={JdStressDeeplinkContent}");
- log.AppendLine($"scheduled={scheduledCount}");
- log.AppendLine($"completed={completedCount}");
- log.AppendLine($"success={successCount}");
- log.AppendLine($"fail={failCount}");
- log.AppendLine($"exception={exceptionCount}");
- log.AppendLine($"riskControl={riskControlCount}");
- log.AppendLine($"peakActive={peakActiveCount}");
- log.AppendLine("messageCounts=");
- foreach (var item in messageCounts.OrderByDescending(item => item.Value))
- {
- log.AppendLine($"{item.Key}\t{item.Value}");
- }
- log.AppendLine("codeCounts=");
- foreach (var item in codeCounts.OrderByDescending(item => item.Value))
- {
- log.AppendLine($"{item.Key}\t{item.Value}");
- }
- log.AppendLine("responses=");
- while (responseLines.TryDequeue(out string? line))
- {
- log.AppendLine(line);
- }
- await log.SaveAsync();
- }
- catch (Exception ex)
- {
- logStatus = ex.Message;
- }
- return new JdStressRunResult
- {
- success = true,
- message = message,
- runId = runId,
- requestPerSecond = t,
- timeSeconds = time,
- expectedRequests = (long)t * time,
- scheduledCount = scheduledCount,
- completedCount = completedCount,
- successCount = successCount,
- failCount = failCount,
- exceptionCount = exceptionCount,
- riskControlCount = riskControlCount,
- peakActiveCount = peakActiveCount,
- averageElapsedMs = completedCount == 0 ? 0 : Math.Round(totalElapsedMs / (double)completedCount, 2),
- elapsedSeconds = Math.Round(stopwatch.Elapsed.TotalSeconds, 2),
- messageCounts = messageCounts.OrderByDescending(item => item.Value).ToDictionary(item => item.Key, item => item.Value),
- codeCounts = codeCounts.OrderByDescending(item => item.Value).ToDictionary(item => item.Key, item => item.Value),
- log = new JdStressLogInfo
- {
- type = "LoggerLibrary",
- dir = "jd_api_stress",
- name = logName,
- status = logStatus
- }
- };
- }
- private static async Task ObserveCompletedTasksAsync(List<Task> runningTasks)
- {
- for (int i = runningTasks.Count - 1; i >= 0; i--)
- {
- if (!runningTasks[i].IsCompleted) continue;
- await runningTasks[i];
- runningTasks.RemoveAt(i);
- }
- }
- private static async Task<JdStressRequestResult> ExecuteJdStressRequestAsync(long sequence, string ip, string oaid, CancellationToken cancellationToken)
- {
- var sw = Stopwatch.StartNew();
- try
- {
- var request = new UnionParseRequest
- {
- Content = JdStressDeeplinkContent,
- Channel = "jd",
- CommerceType = 0,
- Ip = ip,
- Oaid = oaid,
- RiskStrategy = string.Empty,
- LaunchScene = 0,
- AccountId = 0,
- SpecialText = 0,
- QueryText = string.Empty,
- ClickId = string.Empty,
- Type = "dp",
- Pic = string.Empty
- };
- var result = await UnionParseCore.DeeplinkJdParseAsync(request, cancellationToken);
- sw.Stop();
- string response = result.Content ?? string.Empty;
- var parsed = ParseJdStressResponse(response);
- parsed.Sequence = sequence;
- parsed.ElapsedMs = sw.ElapsedMilliseconds;
- parsed.Response = response;
- parsed.FinishedAt = DateTime.Now;
- return parsed;
- }
- catch (Exception ex)
- {
- sw.Stop();
- return new JdStressRequestResult
- {
- Sequence = sequence,
- ElapsedMs = sw.ElapsedMilliseconds,
- Success = false,
- Message = "exception",
- SubMessage = ex.Message,
- Code = 0,
- Response = ex.ToString(),
- FinishedAt = DateTime.Now,
- IsException = true
- };
- }
- }
- private static JdStressRequestResult ParseJdStressResponse(string response)
- {
- var result = new JdStressRequestResult();
- if (string.IsNullOrWhiteSpace(response))
- {
- result.Message = "empty response";
- return result;
- }
- try
- {
- using var doc = JsonDocument.Parse(response);
- var root = doc.RootElement;
- result.Success = ReadBool(root, "success");
- result.Message = ReadString(root, "message");
- result.SubMessage = ReadString(root, "sub_message");
- result.Code = ReadInt(root, "code");
- }
- catch (Exception ex)
- {
- result.Success = false;
- result.Message = "parse response error";
- result.SubMessage = ex.Message;
- result.Code = 0;
- }
- return result;
- }
- private static bool ReadBool(JsonElement root, string propertyName)
- {
- if (!root.TryGetProperty(propertyName, out var property)) return false;
- return property.ValueKind switch
- {
- JsonValueKind.True => true,
- JsonValueKind.False => false,
- JsonValueKind.Number => property.TryGetInt32(out int value) && value != 0,
- JsonValueKind.String => bool.TryParse(property.GetString(), out bool value) && value,
- _ => false
- };
- }
- private static int ReadInt(JsonElement root, string propertyName)
- {
- if (!root.TryGetProperty(propertyName, out var property)) return 0;
- return property.ValueKind switch
- {
- JsonValueKind.Number => property.TryGetInt32(out int value) ? value : 0,
- JsonValueKind.String => int.TryParse(property.GetString(), out int value) ? value : 0,
- _ => 0
- };
- }
- private static string ReadString(JsonElement root, string propertyName)
- {
- if (!root.TryGetProperty(propertyName, out var property)) return string.Empty;
- if (property.ValueKind == JsonValueKind.Null || property.ValueKind == JsonValueKind.Undefined) return string.Empty;
- return property.ValueKind == JsonValueKind.String ? property.GetString() ?? string.Empty : property.ToString();
- }
- private static bool IsRiskControlResult(JdStressRequestResult result)
- {
- if (!"放弃转链".Equals(result.Message, StringComparison.Ordinal)) return false;
- return result.SubMessage.Contains("控制", StringComparison.Ordinal)
- || result.SubMessage.Contains("风控", StringComparison.Ordinal)
- || result.SubMessage.Contains("限流", StringComparison.Ordinal)
- || result.SubMessage.Contains("频", StringComparison.Ordinal)
- || result.SubMessage.Contains("系统繁忙", StringComparison.Ordinal);
- }
- private static string NormalizeLogValue(string value)
- {
- return (value ?? string.Empty)
- .Replace("\r", "\\r")
- .Replace("\n", "\\n")
- .Replace("\t", " ");
- }
- private static void UpdateMax(ref int target, int value)
- {
- int snapshot;
- while (value > (snapshot = Volatile.Read(ref target))
- && Interlocked.CompareExchange(ref target, value, snapshot) != snapshot)
- {
- }
- }
- private sealed class JdStressRequestResult
- {
- public long Sequence { get; set; }
- public bool Success { get; set; }
- public string Message { get; set; } = string.Empty;
- public string SubMessage { get; set; } = string.Empty;
- public int Code { get; set; }
- public long ElapsedMs { get; set; }
- public string Response { get; set; } = string.Empty;
- public DateTime FinishedAt { get; set; }
- public bool IsException { get; set; }
- }
- private sealed class JdStressRunResult
- {
- public bool success { get; set; }
- public string message { get; set; } = string.Empty;
- public string runId { get; set; } = string.Empty;
- public int requestPerSecond { get; set; }
- public int timeSeconds { get; set; }
- public long expectedRequests { get; set; }
- public long scheduledCount { get; set; }
- public long completedCount { get; set; }
- public long successCount { get; set; }
- public long failCount { get; set; }
- public long exceptionCount { get; set; }
- public long riskControlCount { get; set; }
- public int peakActiveCount { get; set; }
- public double averageElapsedMs { get; set; }
- public double elapsedSeconds { get; set; }
- public Dictionary<string, int> messageCounts { get; set; } = [];
- public Dictionary<string, int> codeCounts { get; set; } = [];
- public JdStressLogInfo log { get; set; } = new();
- }
- private sealed class JdStressLogInfo
- {
- public string type { get; set; } = string.Empty;
- public string dir { get; set; } = string.Empty;
- public string name { get; set; } = string.Empty;
- public string status { get; set; } = string.Empty;
- }
- [HttpGet]
- public async Task<ActionResult> xxx()
- {
- var list = await TkPoolCore.ListAsync();
- if (list == null) return new APIResult(new { success = false, message = "没有有效账号", });
- string message = string.Empty;
- foreach (var account in list)
- {
- try
- {
- var alimama = new AlimamaPlus(account);
- alimama.RenewCookie();
- }
- catch (Exception ex)
- {
- message = $"【cookie续期】xxxx\n{ex.Message}\n{ex.StackTrace}";
- NotifyCore.Notify(new NifyMessage
- {
- message = message,
- priority = NifyMessagePriority.high,
- tags = ["red_circle"]
- });
- continue;
- }
- }
- return new APIResult(new { success = true, message = "ok" });
- }
- }
- }
|