using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc.Controllers; using Microsoft.AspNetCore.Mvc.Filters; using System; using System.Collections.Generic; using System.Linq; using System.Text; using dodohold.core; using static dodohold.core.ZTOExpress.CreateOrderArgs; using System.Net; using System.Security.Cryptography; using Spire.Pdf.Exporting.XPS.Schema; using System.Xml.Linq; using static QRCoder.PayloadGenerator; using TencentCloud.Ssl.V20191205.Models; using CSRedis; namespace molilian.core { public class TkLogCore { static string queue_tb_key = "queue:logs:tb"; static string queue_jd_key = "queue:logs:jd"; static string queue_parse_tb_key = "queue:parse_logs:tb"; static string queue_parse_jd_key = "queue:parse_logs:jd"; static string queue_parse_dy_key = "queue:parse_logs:dy"; static string queue_parse_tool_key = "queue:parse_logs:tool"; static string queue_coupon_key = "queue:coupon_logs"; public static int BatchInsertLogDB(int limit) { var result = EndPointCore.ProcessEndPointNodes(node => { if (!node.is_public_api) return 0; var redis = RedisClientManager.GetRedisClient(node.redis_server); return BatchInsertLogDB(limit, redis); }); return result.Sum(); } public static int BatchInsertLogDB(int limit, CSRedisClient redis) { int total = 0; using var connection = DBContext.GetOpenConnection(); connection.Open(); using var transaction = connection.BeginTransaction(); try { for (int i = 0; i < limit; i++) { var data = redis.LPop(queue_tb_key); if (data == null) break; data.id = new DBContext.Table(connection, "tk_logs") .Add("end_point", data.end_point) .Add("channel", (int)data.channel) .Add("accountId", data.accountId) .Add("accountName", data.accountName) .Add("rawContent", data.rawContent) .Add("rawContent2", data.rawContent2) .Add("success", data.success) .Add("message", data.message) .Add("reason", data.reason) .Add("content", data.content) .Add("couponAmount", data.couponAmount) .Add("itemId", data.itemId) .Add("itemName", data.itemName) .Add("pic", data.pic) .Add("promotionPrice", data.promotionPrice) .Add("taoToken", data.taoToken) .Add("shortLinkurl", data.shortLinkurl) .Add("deeplink_url", data.deeplink_url) .Add("num_iid", data.num_iid) .Add("elapsedTime", data.elapsedTime) .Add("ip", data.ip) .Add("oaid", data.oaid) .Add("create_time", data.create_time) .Create(DBContext.InsertType.NORMAL, transaction); if (!string.IsNullOrEmpty(data.itemId)) TkOrderTrackingCore.SaveLinkSummary(data); total++; } for (int i = 0; i < limit; i++) { var data = redis.LPop(queue_jd_key); if (data == null) break; new DBContext.Table(connection, "tk_logs") .Add("end_point", data.end_point) .Add("channel", (int)data.channel) .Add("accountId", data.accountId) .Add("accountName", data.accountName) .Add("rawContent", data.rawContent) .Add("rawContent2", data.rawContent2) .Add("success", data.success) .Add("message", data.message) .Add("reason", data.reason) .Add("shortLinkurl", data.shortLinkurl) .Add("deeplink_url", data.deeplink_url) .Add("elapsedTime", data.elapsedTime) .Add("ip", data.ip) .Add("oaid", data.oaid) .Add("create_time", data.create_time) .Create(DBContext.InsertType.NORMAL, transaction); total++; } for (int i = 0; i < limit; i++) { var data = redis.LPop(queue_parse_tb_key); if (data == null) break; data.id = new DBContext.Table(connection, "tk_parse_logs") .Add("end_point", data.end_point) .Add("channel", (int)data.channel) .Add("accountId", data.accountId) .Add("accountName", data.accountName) .Add("rawContent", data.rawContent) .Add("success", data.success) .Add("message", data.message) .Add("reason", data.reason) .Add("content", data.content) .Add("itemId", data.itemId) .Add("itemName", data.itemName) .Add("pic", data.pic) .Add("couponAmount", data.couponAmount) .Add("promotionPrice", data.promotionPrice) .Add("taoToken", data.taoToken) .Add("shortLinkurl", data.shortLinkurl) .Add("deeplink_url", data.deeplink_url) .Add("num_iid", data.num_iid) .Add("elapsedTime", data.elapsedTime) .Add("elapsedTime2", data.elapsedTime2) .Add("elapsedTime3", data.elapsedTime3) .Add("ip", data.ip) .Add("oaid", data.oaid) .Add("create_time", data.create_time) .Create(DBContext.InsertType.NORMAL, transaction); if (!string.IsNullOrEmpty(data.itemId)) TkOrderTrackingCore.SaveLinkSummary(data); total++; } for (int i = 0; i < limit; i++) { var data = redis.LPop(queue_parse_jd_key); if (data == null) break; new DBContext.Table(connection, "tk_parse_logs") .Add("end_point", data.end_point) .Add("channel", (int)data.channel) .Add("accountId", data.accountId) .Add("accountName", data.accountName) .Add("rawContent", data.rawContent) .Add("success", data.success) .Add("message", data.message) .Add("reason", data.reason) .Add("content", data.content) .Add("itemId", data.itemId) .Add("itemName", data.itemName) .Add("pic", data.pic) .Add("couponAmount", data.couponAmount) .Add("promotionPrice", data.promotionPrice) .Add("taoToken", data.taoToken) .Add("shortLinkurl", data.shortLinkurl) .Add("deeplink_url", data.deeplink_url) .Add("elapsedTime", data.elapsedTime) .Add("ip", data.ip) .Add("oaid", data.oaid) .Add("create_time", data.create_time) .Create(DBContext.InsertType.NORMAL, transaction); total++; } for (int i = 0; i < limit; i++) { var data = redis.LPop(queue_parse_dy_key); if (data == null) break; new DBContext.Table(connection, "tk_parse_logs") .Add("end_point", data.end_point) .Add("channel", (int)data.channel) .Add("accountId", data.accountId) .Add("accountName", data.accountName) .Add("rawContent", data.rawContent) .Add("success", data.success) .Add("message", data.message) .Add("reason", data.reason) .Add("content", data.content) .Add("itemId", data.itemId) .Add("itemName", data.itemName) .Add("pic", data.pic) .Add("couponAmount", data.couponAmount) .Add("promotionPrice", data.promotionPrice) .Add("taoToken", data.taoToken) .Add("shortLinkurl", data.shortLinkurl) .Add("deeplink_url", data.deeplink_url) .Add("elapsedTime", data.elapsedTime) .Add("ip", data.ip) .Add("oaid", data.oaid) .Add("create_time", data.create_time) .Create(DBContext.InsertType.NORMAL, transaction); total++; } for (int i = 0; i < limit; i++) { var data = redis.LPop(queue_parse_tool_key); if (data == null) break; new DBContext.Table(connection, "tool_parse_logs") .Add("end_point", data.end_point) .Add("channel", (int)data.channel) .Add("rawContent", data.rawContent) .Add("success", data.success) .Add("message", data.message) .Add("reason", data.reason) .Add("content", data.content) .Add("taoToken", data.taoToken) .Add("shortLinkurl", data.shortLinkurl) .Add("deeplink_url", data.deeplink_url) .Add("elapsedTime", data.elapsedTime) .Add("ip", data.ip) .Add("oaid", data.oaid) .Add("create_time", data.create_time) .Create(DBContext.InsertType.NORMAL, transaction); total++; } for (int i = 0; i < limit; i++) { var data = redis.LPop(queue_coupon_key); if (data == null) break; connection.Insert(data); total++; } transaction.Commit(); } catch (Exception ex) { transaction.Rollback(); _ = new LoggerLibrary("database_error", "parse_log") .Info(ex.Message, ex.StackTrace) .SaveAsync(); NotifyCore.Notify(new NifyMessage { message = $"【写入日志异常】\n{ex.Message}\n{ex.StackTrace}", priority = NifyMessagePriority.high, tags = ["red_circle"] }); } finally { connection.Close(); } return total; } public static async Task LogAsync(JdDataDTO response) { try { var ts = DateTime.Now - response.create_time; response.elapsedTime = (int)ts.TotalMilliseconds; RedisHelper.RPush(queue_jd_key, response); saveCache(response.channel.ToString(), response.accountId, response.accountName, response.success, response.message, response.reason); } catch (Exception ex) { } } public static async Task LogAsync(TkDataDTO response, AlimamaPlus? alimamaPlus = null) { try { var ts = DateTime.Now - response.create_time; response.elapsedTime = (int)ts.TotalMilliseconds; RedisHelper.RPush(queue_tb_key, response); saveCache(response.channel.ToString(), response.accountId, response.accountName, response.success, response.message, response.reason); if (!response.success && "nologin".Equals(response.message)) { switch (response.channel) { case TkChannelEnum.tb: await Task.Run(() => { if (alimamaPlus != null) { (bool success, string message) = alimamaPlus.RenewCookie(); if (success) return; } TkPoolCore.Disabled(response.accountName, $"{response.rawContent}\n{response.rawContent2}"); }); break; } } if ("没有匹配账号".Equals(response.reason)) { TkPoolCore.AccountExhausted(); } } catch (Exception ex) { _ = new LoggerLibrary("unionParse", "database_error") .Info(response.rawContent, response.rawContent2) .Info(response.Convert2Json()) .Info(ex.Message, ex.StackTrace) .SaveAsync(); } } public static async Task ParseLogAsync(TkDataDTO response, AlimamaPlus? alimamaPlus = null) { #if DEBUG //return; #endif try { var ts = DateTime.Now - response.create_time; response.elapsedTime = (int)ts.TotalMilliseconds; RedisHelper.RPush(queue_parse_tb_key, response); saveParseCache(response.channel.ToString(), response.accountId, response.accountName, response.success, response.message, response.reason); if (!string.IsNullOrEmpty(response.itemId)) TkOrderTrackingCore.SaveLinkSummary(response); if (!response.success && "nologin".Equals(response.message)) { switch (response.channel) { case TkChannelEnum.tb: await Task.Run(() => { if (alimamaPlus != null) { (bool success, string message) = alimamaPlus.RenewCookie(); if (success) return; } TkPoolCore.Disabled(response.accountName, $"{response.rawContent}"); }); break; } } } catch (Exception ex) { _ = new LoggerLibrary("unionParse", "database_error") .Info(response.rawContent) .Info(response.Convert2Json()) .Info(ex.Message, ex.StackTrace) .SaveAsync(); } } public static async Task CouponLogAsync(UnionCouponDTO response, AlimamaPlus? alimamaPlus = null) { try { var ts = DateTime.Now - response.create_time; response.elapsedTime = (int)ts.TotalMilliseconds; RedisHelper.RPush(queue_coupon_key, response); saveCouponCache(response.channel.ToString(), response.accountId, response.accountName, response.success, response.message, response.reason); if (!response.success && "nologin".Equals(response.message)) { switch (response.channel) { case TkChannelEnum.tb: await Task.Run(() => { if (alimamaPlus != null) { (bool success, string message) = alimamaPlus.RenewCookie(); if (success) return; } TkPoolCore.Disabled(response.accountName, $"{response.rawContent}"); }); break; } } } catch (Exception ex) { _ = new LoggerLibrary("unionCoupon", "database_error") .Info(response.rawContent) .Info(response.Convert2Json()) .Info(ex.Message, ex.StackTrace) .SaveAsync(); } } public static async Task ParseLogAsync(JdDataDTO response) { try { var ts = DateTime.Now - response.create_time; response.elapsedTime = (int)ts.TotalMilliseconds; RedisHelper.RPush(queue_parse_jd_key, response); saveParseCache(response.channel.ToString(), response.accountId, response.accountName, response.success, response.message, response.reason); } catch (Exception ex) { } } public static async Task ParseLogAsync(DyDataDTO response) { try { var ts = DateTime.Now - response.create_time; response.elapsedTime = (int)ts.TotalMilliseconds; RedisHelper.RPush(queue_parse_dy_key, response); saveParseCache(response.channel.ToString(), response.accountId, response.accountName, response.success, response.message, response.reason); } catch (Exception ex) { } } public static async Task ParseLogAsync(ToolParseDataDTO response) { try { var ts = DateTime.Now - response.create_time; response.elapsedTime = (int)ts.TotalMilliseconds; RedisHelper.RPush(queue_parse_tool_key, response); saveParseCache(response.channel.ToString(), 0, "tool", response.success, response.message, response.reason); } catch (Exception ex) { } } private static void saveCache(string channel, int accountId, string accountName, bool success, string message, string reason) { saveAccountCache("all", success, message, reason); saveAccountCache($"{channel}", success, message, reason); saveAccountCache($"{accountName}", success, message, reason); if (accountId != 0) { //todo 放着跑两天，要将读取的地方改成读取accountid saveAccountCache($"{channel}_{accountId}", success, message, reason); } } private static void saveAccountCache(string accountName, bool success, string message, string reason) { RedisHelper.IncrBy($":total:{accountName}:{DateTime.Now:yyyyMM}"); RedisHelper.IncrBy($":total:{accountName}:{DateTime.Now:yyyyMMdd}"); RedisHelper.IncrBy($":total:{accountName}:{DateTime.Now:yyyyMMddHH}"); string result = success ? "success" : "fail"; RedisHelper.IncrBy($":total:{accountName}:{result}:{DateTime.Now:yyyyMM}"); RedisHelper.IncrBy($":total:{accountName}:{result}:{DateTime.Now:yyyyMMdd}"); RedisHelper.IncrBy($":total:{accountName}:{result}:{DateTime.Now:yyyyMMddHH}"); if (!string.IsNullOrEmpty(message)) { RedisHelper.SAdd($":total:{accountName}:message:{DateTime.Now:yyyyMM}", message); RedisHelper.SAdd($":total:{accountName}:message:{DateTime.Now:yyyyMMdd}", message); RedisHelper.SAdd($":total:{accountName}:message:{DateTime.Now:yyyyMMddHH}", message); RedisHelper.IncrBy($":total:{accountName}:{message}:{DateTime.Now:yyyyMM}"); RedisHelper.IncrBy($":total:{accountName}:{message}:{DateTime.Now:yyyyMMdd}"); RedisHelper.IncrBy($":total:{accountName}:{message}:{DateTime.Now:yyyyMMddHH}"); } if (!string.IsNullOrEmpty(reason)) { RedisHelper.SAdd($":total:{accountName}:reason:{DateTime.Now:yyyyMM}", reason); RedisHelper.SAdd($":total:{accountName}:reason:{DateTime.Now:yyyyMMdd}", reason); RedisHelper.SAdd($":total:{accountName}:reason:{DateTime.Now:yyyyMMddHH}", reason); RedisHelper.IncrBy($":total:{accountName}:{reason}:{DateTime.Now:yyyyMM}"); RedisHelper.IncrBy($":total:{accountName}:{reason}:{DateTime.Now:yyyyMMdd}"); RedisHelper.IncrBy($":total:{accountName}:{reason}:{DateTime.Now:yyyyMMddHH}"); } } private static void saveParseCache(string channel, int accountId, string accountName, bool success, string message, string reason) { saveParseAccountCache("all", success, message, reason); saveParseAccountCache($"{channel}", success, message, reason); saveParseAccountCache($"{accountName}", success, message, reason); if (accountId != 0) { //todo 放着跑两天，要将读取的地方改成读取accountid saveParseAccountCache($"{channel}_{accountId}", success, message, reason); } } private static void saveParseAccountCache(string accountName, bool success, string message, string reason) { RedisHelper.IncrBy($":parse_total:{accountName}:{DateTime.Now:yyyyMM}"); RedisHelper.IncrBy($":parse_total:{accountName}:{DateTime.Now:yyyyMMdd}"); RedisHelper.IncrBy($":parse_total:{accountName}:{DateTime.Now:yyyyMMddHH}"); string result = success ? "success" : "fail"; RedisHelper.IncrBy($":parse_total:{accountName}:{result}:{DateTime.Now:yyyyMM}"); RedisHelper.IncrBy($":parse_total:{accountName}:{result}:{DateTime.Now:yyyyMMdd}"); RedisHelper.IncrBy($":parse_total:{accountName}:{result}:{DateTime.Now:yyyyMMddHH}"); if (!string.IsNullOrEmpty(message)) { RedisHelper.SAdd($":parse_total:{accountName}:message:{DateTime.Now:yyyyMM}", message); RedisHelper.SAdd($":parse_total:{accountName}:message:{DateTime.Now:yyyyMMdd}", message); RedisHelper.SAdd($":parse_total:{accountName}:message:{DateTime.Now:yyyyMMddHH}", message); RedisHelper.IncrBy($":parse_total:{accountName}:{message}:{DateTime.Now:yyyyMM}"); RedisHelper.IncrBy($":parse_total:{accountName}:{message}:{DateTime.Now:yyyyMMdd}"); RedisHelper.IncrBy($":parse_total:{accountName}:{message}:{DateTime.Now:yyyyMMddHH}"); } if (!string.IsNullOrEmpty(reason)) { RedisHelper.SAdd($":parse_total:{accountName}:reason:{DateTime.Now:yyyyMM}", reason); RedisHelper.SAdd($":parse_total:{accountName}:reason:{DateTime.Now:yyyyMMdd}", reason); RedisHelper.SAdd($":parse_total:{accountName}:reason:{DateTime.Now:yyyyMMddHH}", reason); RedisHelper.IncrBy($":parse_total:{accountName}:{reason}:{DateTime.Now:yyyyMM}"); RedisHelper.IncrBy($":parse_total:{accountName}:{reason}:{DateTime.Now:yyyyMMdd}"); RedisHelper.IncrBy($":parse_total:{accountName}:{reason}:{DateTime.Now:yyyyMMddHH}"); } } private static void saveCouponCache(string channel, int accountId, string accountName, bool success, string message, string reason) { saveAccountCouponCache("all", success, message, reason); saveAccountCouponCache($"{channel}", success, message, reason); if (accountId != 0) { //todo 放着跑两天，要将读取的地方改成读取accountid saveAccountCouponCache($"{channel}_{accountId}", success, message, reason); } } private static void saveAccountCouponCache(string accountName, bool success, string message, string reason) { RedisHelper.IncrBy($":coupon_total:{accountName}:{DateTime.Now:yyyyMM}"); RedisHelper.IncrBy($":coupon_total:{accountName}:{DateTime.Now:yyyyMMdd}"); RedisHelper.IncrBy($":coupon_total:{accountName}:{DateTime.Now:yyyyMMddHH}"); string result = success ? "success" : "fail"; RedisHelper.IncrBy($":coupon_total:{accountName}:{result}:{DateTime.Now:yyyyMM}"); RedisHelper.IncrBy($":coupon_total:{accountName}:{result}:{DateTime.Now:yyyyMMdd}"); RedisHelper.IncrBy($":coupon_total:{accountName}:{result}:{DateTime.Now:yyyyMMddHH}"); if (!string.IsNullOrEmpty(message)) { RedisHelper.SAdd($":coupon_total:{accountName}:message:{DateTime.Now:yyyyMM}", message); RedisHelper.SAdd($":coupon_total:{accountName}:message:{DateTime.Now:yyyyMMdd}", message); RedisHelper.SAdd($":coupon_total:{accountName}:message:{DateTime.Now:yyyyMMddHH}", message); RedisHelper.IncrBy($":coupon_total:{accountName}:{message}:{DateTime.Now:yyyyMM}"); RedisHelper.IncrBy($":coupon_total:{accountName}:{message}:{DateTime.Now:yyyyMMdd}"); RedisHelper.IncrBy($":coupon_total:{accountName}:{message}:{DateTime.Now:yyyyMMddHH}"); } if (!string.IsNullOrEmpty(reason)) { RedisHelper.SAdd($":coupon_total:{accountName}:reason:{DateTime.Now:yyyyMM}", reason); RedisHelper.SAdd($":coupon_total:{accountName}:reason:{DateTime.Now:yyyyMMdd}", reason); RedisHelper.SAdd($":coupon_total:{accountName}:reason:{DateTime.Now:yyyyMMddHH}", reason); RedisHelper.IncrBy($":coupon_total:{accountName}:{reason}:{DateTime.Now:yyyyMM}"); RedisHelper.IncrBy($":coupon_total:{accountName}:{reason}:{DateTime.Now:yyyyMMdd}"); RedisHelper.IncrBy($":coupon_total:{accountName}:{reason}:{DateTime.Now:yyyyMMddHH}"); } } public static int GetTotal(string keyname, bool all_node = true) { int result = RedisHelper.Get(keyname); return result; } public static string[] GetTotalKeys(string keyname, bool all_node = true) { string[] message_keys = RedisHelper.SMembers(keyname); return message_keys; //string[] message_keys2 = RedisHelper.SMembers(keyname.Replace(":total:", ":parse_total:")); //message_keys = message_keys.Union(message_keys2).ToArray(); //string[] message_keys3 = RedisHelper.SMembers(keyname.Replace(":total:", ":coupon_total:")); //message_keys = message_keys.Union(message_keys3).ToArray(); } //if (item.total_count > 0) continue; //item.total_count = RedisHelper.Get($":total:all:{item.report_date:yyyyMMdd}"); //if (item.total_count == 0) continue; //item.success_count = RedisHelper.Get($":total:all:success:{item.report_date:yyyyMMdd}"); //item.abandon_count = RedisHelper.Get($":total:all:放弃转链:{item.report_date:yyyyMMdd}"); } }