dotnet/molilian.core/Core/taoke/TkLogCore.cs

using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Mvc.Controllers;
using Microsoft.AspNetCore.Mvc.Filters;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using dodohold.core;
using static dodohold.core.ZTOExpress.CreateOrderArgs;
using System.Net;
using System.Security.Cryptography;
using Spire.Pdf.Exporting.XPS.Schema;
using System.Xml.Linq;
using static QRCoder.PayloadGenerator;
using TencentCloud.Ssl.V20191205.Models;
using CSRedis;


namespace molilian.core
{
    public class TkLogCore
    {
        static string queue_tb_key = "queue:logs:tb";
        static string queue_jd_key = "queue:logs:jd";

        static string queue_parse_tb_key = "queue:parse_logs:tb";
        static string queue_parse_jd_key = "queue:parse_logs:jd";
        static string queue_parse_dy_key = "queue:parse_logs:dy";
        static string queue_parse_tool_key = "queue:parse_logs:tool";

        static string queue_coupon_key = "queue:coupon_logs";

        public static int BatchInsertLogDB(int limit)
        {
            var result = EndPointCore.ProcessEndPointNodes<int>(node =>
            {
                if (!node.is_public_api) return 0;
                var redis = RedisClientManager.GetRedisClient(node.redis_server);
                return BatchInsertLogDB(limit, redis);
            });
            return result.Sum();
        }


        public static int BatchInsertLogDB(int limit, CSRedisClient redis)
        {
            int total = 0;
            using var connection = DBContext.GetOpenConnection();
            connection.Open();
            using var transaction = connection.BeginTransaction();
            try
            {
                for (int i = 0; i < limit; i++)
                {
                    var data = redis.LPop<TkDataDTO>(queue_tb_key);
                    if (data == null) break;
                    data.id = new DBContext.Table(connection, "tk_logs")
                        .Add("end_point", data.end_point)
                        .Add("channel", (int)data.channel)
                        .Add("accountId", data.accountId)
                        .Add("accountName", data.accountName)
                        .Add("rawContent", data.rawContent)
                        .Add("rawContent2", data.rawContent2)
                        .Add("success", data.success)
                        .Add("message", data.message)
                        .Add("reason", data.reason)
                        .Add("content", data.content)
                        .Add("couponAmount", data.couponAmount)
                        .Add("itemId", data.itemId)
                        .Add("itemName", data.itemName)
                        .Add("pic", data.pic)
                        .Add("promotionPrice", data.promotionPrice)
                        .Add("taoToken", data.taoToken)
                        .Add("shortLinkurl", data.shortLinkurl)
                        .Add("deeplink_url", data.deeplink_url)
                        .Add("num_iid", data.num_iid)
                        .Add("elapsedTime", data.elapsedTime)
                        .Add("ip", data.ip)
                        .Add("oaid", data.oaid)
                        .Add("create_time", data.create_time)
                        .Create(DBContext.InsertType.NORMAL, transaction);
                    if (!string.IsNullOrEmpty(data.itemId)) TkOrderTrackingCore.SaveLinkSummary(data);
                    total++;
                }

                for (int i = 0; i < limit; i++)
                {
                    var data = redis.LPop<JdDataDTO>(queue_jd_key);
                    if (data == null) break;
                    new DBContext.Table(connection, "tk_logs")
                        .Add("end_point", data.end_point)
                        .Add("channel", (int)data.channel)
                        .Add("accountId", data.accountId)
                        .Add("accountName", data.accountName)
                        .Add("rawContent", data.rawContent)
                        .Add("rawContent2", data.rawContent2)
                        .Add("success", data.success)
                        .Add("message", data.message)
                        .Add("reason", data.reason)
                        .Add("shortLinkurl", data.shortLinkurl)
                        .Add("deeplink_url", data.deeplink_url)
                        .Add("elapsedTime", data.elapsedTime)
                        .Add("ip", data.ip)
                        .Add("oaid", data.oaid)
                        .Add("create_time", data.create_time)
                        .Create(DBContext.InsertType.NORMAL, transaction);
                    total++;
                }

                for (int i = 0; i < limit; i++)
                {
                    var data = redis.LPop<TkDataDTO>(queue_parse_tb_key);
                    if (data == null) break;

                    data.id = new DBContext.Table(connection, "tk_parse_logs")
                        .Add("end_point", data.end_point)
                        .Add("channel", (int)data.channel)
                        .Add("accountId", data.accountId)
                        .Add("accountName", data.accountName)
                        .Add("rawContent", data.rawContent)
                        .Add("success", data.success)
                        .Add("message", data.message)
                        .Add("reason", data.reason)
                        .Add("content", data.content)
                        .Add("itemId", data.itemId)
                        .Add("itemName", data.itemName)
                        .Add("pic", data.pic)
                        .Add("couponAmount", data.couponAmount)
                        .Add("promotionPrice", data.promotionPrice)
                        .Add("taoToken", data.taoToken)
                        .Add("shortLinkurl", data.shortLinkurl)
                        .Add("deeplink_url", data.deeplink_url)
                        .Add("num_iid", data.num_iid)
                        .Add("elapsedTime", data.elapsedTime)
                        .Add("elapsedTime2", data.elapsedTime2)
                        .Add("elapsedTime3", data.elapsedTime3)
                        .Add("ip", data.ip)
                        .Add("oaid", data.oaid)
                        .Add("create_time", data.create_time)
                        .Create(DBContext.InsertType.NORMAL, transaction);
                    if (!string.IsNullOrEmpty(data.itemId)) TkOrderTrackingCore.SaveLinkSummary(data);
                    total++;
                }

                for (int i = 0; i < limit; i++)
                {
                    var data = redis.LPop<JdDataDTO>(queue_parse_jd_key);
                    if (data == null) break;
                    new DBContext.Table(connection, "tk_parse_logs")
                        .Add("end_point", data.end_point)
                        .Add("channel", (int)data.channel)
                        .Add("accountId", data.accountId)
                        .Add("accountName", data.accountName)
                        .Add("rawContent", data.rawContent)
                        .Add("success", data.success)
                        .Add("message", data.message)
                        .Add("reason", data.reason)
                        .Add("content", data.content)
                        .Add("itemId", data.itemId)
                        .Add("itemName", data.itemName)
                        .Add("pic", data.pic)
                        .Add("couponAmount", data.couponAmount)
                        .Add("promotionPrice", data.promotionPrice)
                        .Add("taoToken", data.taoToken)
                        .Add("shortLinkurl", data.shortLinkurl)
                        .Add("deeplink_url", data.deeplink_url)
                        .Add("elapsedTime", data.elapsedTime)
                        .Add("ip", data.ip)
                        .Add("oaid", data.oaid)
                        .Add("create_time", data.create_time)
                        .Create(DBContext.InsertType.NORMAL, transaction);
                    total++;
                }
                for (int i = 0; i < limit; i++)
                {
                    var data = redis.LPop<DyDataDTO>(queue_parse_dy_key);
                    if (data == null) break;
                    new DBContext.Table(connection, "tk_parse_logs")
                        .Add("end_point", data.end_point)
                        .Add("channel", (int)data.channel)
                        .Add("accountId", data.accountId)
                        .Add("accountName", data.accountName)
                        .Add("rawContent", data.rawContent)
                        .Add("success", data.success)
                        .Add("message", data.message)
                        .Add("reason", data.reason)
                        .Add("content", data.content)
                        .Add("itemId", data.itemId)
                        .Add("itemName", data.itemName)
                        .Add("pic", data.pic)
                        .Add("couponAmount", data.couponAmount)
                        .Add("promotionPrice", data.promotionPrice)
                        .Add("taoToken", data.taoToken)
                        .Add("shortLinkurl", data.shortLinkurl)
                        .Add("deeplink_url", data.deeplink_url)
                        .Add("elapsedTime", data.elapsedTime)
                        .Add("ip", data.ip)
                        .Add("oaid", data.oaid)
                        .Add("create_time", data.create_time)
                        .Create(DBContext.InsertType.NORMAL, transaction);
                    total++;
                }
                for (int i = 0; i < limit; i++)
                {
                    var data = redis.LPop<ToolParseDataDTO>(queue_parse_tool_key);
                    if (data == null) break;
                    new DBContext.Table(connection, "tool_parse_logs")
                        .Add("end_point", data.end_point)
                        .Add("channel", (int)data.channel)
                        .Add("rawContent", data.rawContent)
                        .Add("success", data.success)
                        .Add("message", data.message)
                        .Add("reason", data.reason)
                        .Add("content", data.content)
                        .Add("taoToken", data.taoToken)
                        .Add("shortLinkurl", data.shortLinkurl)
                        .Add("deeplink_url", data.deeplink_url)
                        .Add("elapsedTime", data.elapsedTime)
                        .Add("ip", data.ip)
                        .Add("oaid", data.oaid)
                        .Add("create_time", data.create_time)
                        .Create(DBContext.InsertType.NORMAL, transaction);
                    total++;
                }


                for (int i = 0; i < limit; i++)
                {
                    var data = redis.LPop<UnionCouponDTO>(queue_coupon_key);
                    if (data == null) break;
                    connection.Insert(data);
                    total++;
                }
                transaction.Commit();
            }
            catch (Exception ex)
            {
                transaction.Rollback();

                _ = new LoggerLibrary("database_error", "parse_log")
                    .Info(ex.Message, ex.StackTrace)
                    .SaveAsync();

                NotifyCore.Notify(new NifyMessage
                {
                    message = $"【写入日志异常】\n{ex.Message}\n{ex.StackTrace}",
                    priority = NifyMessagePriority.high,
                    tags = ["red_circle"]
                });
            }
            finally
            {
                connection.Close();
            }
            return total;
        }

        public static async Task LogAsync(JdDataDTO response)
        {
            try
            {
                var ts = DateTime.Now - response.create_time;
                response.elapsedTime = (int)ts.TotalMilliseconds;
                RedisHelper.RPush(queue_jd_key, response);
                saveCache(response.channel.ToString(), response.accountId, response.accountName, response.success, response.message, response.reason);
            }
            catch (Exception ex) { }
        }

        public static async Task LogAsync(TkDataDTO response, AlimamaPlus? alimamaPlus = null)
        {
            try
            {
                var ts = DateTime.Now - response.create_time;
                response.elapsedTime = (int)ts.TotalMilliseconds;
                RedisHelper.RPush(queue_tb_key, response);
                saveCache(response.channel.ToString(), response.accountId, response.accountName, response.success, response.message, response.reason);

                if (!response.success && "nologin".Equals(response.message))
                {
                    switch (response.channel)
                    {
                        case TkChannelEnum.tb:

                            await Task.Run(() =>
                            {
                                if (alimamaPlus != null)
                                {
                                    (bool success, string message) = alimamaPlus.RenewCookie();
                                    if (success) return;
                                }
                                TkPoolCore.Disabled(response.accountName, $"{response.rawContent}\n{response.rawContent2}");
                            });
                            break;
                    }
                }
                if ("没有匹配账号".Equals(response.reason))
                {
                    TkPoolCore.AccountExhausted();
                }
            }
            catch (Exception ex)
            {
                _ = new LoggerLibrary("unionParse", "database_error")
                .Info(response.rawContent, response.rawContent2)
                .Info(response.Convert2Json())
                .Info(ex.Message, ex.StackTrace)
                .SaveAsync();
            }
        }

        public static async Task ParseLogAsync(TkDataDTO response, AlimamaPlus? alimamaPlus = null)
        {
#if DEBUG
            //return;
#endif
            try
            {
                var ts = DateTime.Now - response.create_time;
                response.elapsedTime = (int)ts.TotalMilliseconds;
                RedisHelper.RPush(queue_parse_tb_key, response);
                saveParseCache(response.channel.ToString(), response.accountId, response.accountName, response.success, response.message, response.reason);
                if (!string.IsNullOrEmpty(response.itemId)) TkOrderTrackingCore.SaveLinkSummary(response);

                if (!response.success && "nologin".Equals(response.message))
                {
                    switch (response.channel)
                    {
                        case TkChannelEnum.tb:
                            await Task.Run(() =>
                            {
                                if (alimamaPlus != null)
                                {
                                    (bool success, string message) = alimamaPlus.RenewCookie();
                                    if (success) return;
                                }
                                TkPoolCore.Disabled(response.accountName, $"{response.rawContent}");
                            }); break;
                    }
                }
            }
            catch (Exception ex)
            {
                _ = new LoggerLibrary("unionParse", "database_error")
                .Info(response.rawContent)
                .Info(response.Convert2Json())
                .Info(ex.Message, ex.StackTrace)
                .SaveAsync();
            }
        }
        public static async Task CouponLogAsync(UnionCouponDTO response, AlimamaPlus? alimamaPlus = null)
        {
            try
            {
                var ts = DateTime.Now - response.create_time;
                response.elapsedTime = (int)ts.TotalMilliseconds;
                RedisHelper.RPush(queue_coupon_key, response);
                saveCouponCache(response.channel.ToString(), response.accountId, response.accountName, response.success, response.message, response.reason);

                if (!response.success && "nologin".Equals(response.message))
                {
                    switch (response.channel)
                    {
                        case TkChannelEnum.tb:
                            await Task.Run(() =>
                            {
                                if (alimamaPlus != null)
                                {
                                    (bool success, string message) = alimamaPlus.RenewCookie();
                                    if (success) return;
                                }
                                TkPoolCore.Disabled(response.accountName, $"{response.rawContent}");
                            }); break;
                    }
                }
            }
            catch (Exception ex)
            {
                _ = new LoggerLibrary("unionCoupon", "database_error")
                .Info(response.rawContent)
                .Info(response.Convert2Json())
                .Info(ex.Message, ex.StackTrace)
                .SaveAsync();
            }
        }
        public static async Task ParseLogAsync(JdDataDTO response)
        {
            try
            {
                var ts = DateTime.Now - response.create_time;
                response.elapsedTime = (int)ts.TotalMilliseconds;
                RedisHelper.RPush(queue_parse_jd_key, response);
                saveParseCache(response.channel.ToString(), response.accountId, response.accountName, response.success, response.message, response.reason);

            }
            catch (Exception ex) { }
        }
        public static async Task ParseLogAsync(DyDataDTO response)
        {
            try
            {
                var ts = DateTime.Now - response.create_time;
                response.elapsedTime = (int)ts.TotalMilliseconds;
                RedisHelper.RPush(queue_parse_dy_key, response);
                saveParseCache(response.channel.ToString(), response.accountId, response.accountName, response.success, response.message, response.reason);

            }
            catch (Exception ex) { }
        }
        public static async Task ParseLogAsync(ToolParseDataDTO response)
        {
            try
            {
                var ts = DateTime.Now - response.create_time;
                response.elapsedTime = (int)ts.TotalMilliseconds;
                RedisHelper.RPush(queue_parse_tool_key, response);
                saveParseCache(response.channel.ToString(), 0, "tool", response.success, response.message, response.reason);
            }
            catch (Exception ex) { }
        }

        private static void saveCache(string channel, int accountId, string accountName, bool success, string message, string reason)
        {
            saveAccountCache("all", success, message, reason);
            saveAccountCache($"{channel}", success, message, reason);
            saveAccountCache($"{accountName}", success, message, reason);
            if (accountId != 0)
            {
                //todo 放着跑两天，要将读取的地方改成读取accountid
                saveAccountCache($"{channel}_{accountId}", success, message, reason);
            }
        }

        private static void saveAccountCache(string accountName, bool success, string message, string reason)
        {
            RedisHelper.IncrBy($":total:{accountName}:{DateTime.Now:yyyyMM}");
            RedisHelper.IncrBy($":total:{accountName}:{DateTime.Now:yyyyMMdd}");
            RedisHelper.IncrBy($":total:{accountName}:{DateTime.Now:yyyyMMddHH}");


            string result = success ? "success" : "fail";
            RedisHelper.IncrBy($":total:{accountName}:{result}:{DateTime.Now:yyyyMM}");
            RedisHelper.IncrBy($":total:{accountName}:{result}:{DateTime.Now:yyyyMMdd}");
            RedisHelper.IncrBy($":total:{accountName}:{result}:{DateTime.Now:yyyyMMddHH}");


            if (!string.IsNullOrEmpty(message))
            {
                RedisHelper.SAdd($":total:{accountName}:message:{DateTime.Now:yyyyMM}", message);
                RedisHelper.SAdd($":total:{accountName}:message:{DateTime.Now:yyyyMMdd}", message);
                RedisHelper.SAdd($":total:{accountName}:message:{DateTime.Now:yyyyMMddHH}", message);

                RedisHelper.IncrBy($":total:{accountName}:{message}:{DateTime.Now:yyyyMM}");
                RedisHelper.IncrBy($":total:{accountName}:{message}:{DateTime.Now:yyyyMMdd}");
                RedisHelper.IncrBy($":total:{accountName}:{message}:{DateTime.Now:yyyyMMddHH}");
            }

            if (!string.IsNullOrEmpty(reason))
            {
                RedisHelper.SAdd($":total:{accountName}:reason:{DateTime.Now:yyyyMM}", reason);
                RedisHelper.SAdd($":total:{accountName}:reason:{DateTime.Now:yyyyMMdd}", reason);
                RedisHelper.SAdd($":total:{accountName}:reason:{DateTime.Now:yyyyMMddHH}", reason);

                RedisHelper.IncrBy($":total:{accountName}:{reason}:{DateTime.Now:yyyyMM}");
                RedisHelper.IncrBy($":total:{accountName}:{reason}:{DateTime.Now:yyyyMMdd}");
                RedisHelper.IncrBy($":total:{accountName}:{reason}:{DateTime.Now:yyyyMMddHH}");
            }

        }

        private static void saveParseCache(string channel, int accountId, string accountName, bool success, string message, string reason)
        {
            saveParseAccountCache("all", success, message, reason);
            saveParseAccountCache($"{channel}", success, message, reason);
            saveParseAccountCache($"{accountName}", success, message, reason);
            if (accountId != 0)
            {
                //todo 放着跑两天，要将读取的地方改成读取accountid
                saveParseAccountCache($"{channel}_{accountId}", success, message, reason);
            }
        }

        private static void saveParseAccountCache(string accountName, bool success, string message, string reason)
        {
            RedisHelper.IncrBy($":parse_total:{accountName}:{DateTime.Now:yyyyMM}");
            RedisHelper.IncrBy($":parse_total:{accountName}:{DateTime.Now:yyyyMMdd}");
            RedisHelper.IncrBy($":parse_total:{accountName}:{DateTime.Now:yyyyMMddHH}");


            string result = success ? "success" : "fail";
            RedisHelper.IncrBy($":parse_total:{accountName}:{result}:{DateTime.Now:yyyyMM}");
            RedisHelper.IncrBy($":parse_total:{accountName}:{result}:{DateTime.Now:yyyyMMdd}");
            RedisHelper.IncrBy($":parse_total:{accountName}:{result}:{DateTime.Now:yyyyMMddHH}");


            if (!string.IsNullOrEmpty(message))
            {
                RedisHelper.SAdd($":parse_total:{accountName}:message:{DateTime.Now:yyyyMM}", message);
                RedisHelper.SAdd($":parse_total:{accountName}:message:{DateTime.Now:yyyyMMdd}", message);
                RedisHelper.SAdd($":parse_total:{accountName}:message:{DateTime.Now:yyyyMMddHH}", message);

                RedisHelper.IncrBy($":parse_total:{accountName}:{message}:{DateTime.Now:yyyyMM}");
                RedisHelper.IncrBy($":parse_total:{accountName}:{message}:{DateTime.Now:yyyyMMdd}");
                RedisHelper.IncrBy($":parse_total:{accountName}:{message}:{DateTime.Now:yyyyMMddHH}");
            }

            if (!string.IsNullOrEmpty(reason))
            {
                RedisHelper.SAdd($":parse_total:{accountName}:reason:{DateTime.Now:yyyyMM}", reason);
                RedisHelper.SAdd($":parse_total:{accountName}:reason:{DateTime.Now:yyyyMMdd}", reason);
                RedisHelper.SAdd($":parse_total:{accountName}:reason:{DateTime.Now:yyyyMMddHH}", reason);

                RedisHelper.IncrBy($":parse_total:{accountName}:{reason}:{DateTime.Now:yyyyMM}");
                RedisHelper.IncrBy($":parse_total:{accountName}:{reason}:{DateTime.Now:yyyyMMdd}");
                RedisHelper.IncrBy($":parse_total:{accountName}:{reason}:{DateTime.Now:yyyyMMddHH}");
            }

        }
        private static void saveCouponCache(string channel, int accountId, string accountName, bool success, string message, string reason)
        {
            saveAccountCouponCache("all", success, message, reason);
            saveAccountCouponCache($"{channel}", success, message, reason);
            if (accountId != 0)
            {
                //todo 放着跑两天，要将读取的地方改成读取accountid
                saveAccountCouponCache($"{channel}_{accountId}", success, message, reason);
            }
        }

        private static void saveAccountCouponCache(string accountName, bool success, string message, string reason)
        {
            RedisHelper.IncrBy($":coupon_total:{accountName}:{DateTime.Now:yyyyMM}");
            RedisHelper.IncrBy($":coupon_total:{accountName}:{DateTime.Now:yyyyMMdd}");
            RedisHelper.IncrBy($":coupon_total:{accountName}:{DateTime.Now:yyyyMMddHH}");


            string result = success ? "success" : "fail";
            RedisHelper.IncrBy($":coupon_total:{accountName}:{result}:{DateTime.Now:yyyyMM}");
            RedisHelper.IncrBy($":coupon_total:{accountName}:{result}:{DateTime.Now:yyyyMMdd}");
            RedisHelper.IncrBy($":coupon_total:{accountName}:{result}:{DateTime.Now:yyyyMMddHH}");


            if (!string.IsNullOrEmpty(message))
            {
                RedisHelper.SAdd($":coupon_total:{accountName}:message:{DateTime.Now:yyyyMM}", message);
                RedisHelper.SAdd($":coupon_total:{accountName}:message:{DateTime.Now:yyyyMMdd}", message);
                RedisHelper.SAdd($":coupon_total:{accountName}:message:{DateTime.Now:yyyyMMddHH}", message);

                RedisHelper.IncrBy($":coupon_total:{accountName}:{message}:{DateTime.Now:yyyyMM}");
                RedisHelper.IncrBy($":coupon_total:{accountName}:{message}:{DateTime.Now:yyyyMMdd}");
                RedisHelper.IncrBy($":coupon_total:{accountName}:{message}:{DateTime.Now:yyyyMMddHH}");
            }

            if (!string.IsNullOrEmpty(reason))
            {
                RedisHelper.SAdd($":coupon_total:{accountName}:reason:{DateTime.Now:yyyyMM}", reason);
                RedisHelper.SAdd($":coupon_total:{accountName}:reason:{DateTime.Now:yyyyMMdd}", reason);
                RedisHelper.SAdd($":coupon_total:{accountName}:reason:{DateTime.Now:yyyyMMddHH}", reason);

                RedisHelper.IncrBy($":coupon_total:{accountName}:{reason}:{DateTime.Now:yyyyMM}");
                RedisHelper.IncrBy($":coupon_total:{accountName}:{reason}:{DateTime.Now:yyyyMMdd}");
                RedisHelper.IncrBy($":coupon_total:{accountName}:{reason}:{DateTime.Now:yyyyMMddHH}");
            }

        }

        public static int GetTotal(string keyname, bool all_node = true)
        {
            int result = RedisHelper.Get<int>(keyname);
            return result;
        }

        public static string[] GetTotalKeys(string keyname, bool all_node = true)
        {
            string[] message_keys = RedisHelper.SMembers(keyname);
            return message_keys;

            //string[] message_keys2 = RedisHelper.SMembers(keyname.Replace(":total:", ":parse_total:"));
            //message_keys = message_keys.Union(message_keys2).ToArray();

            //string[] message_keys3 = RedisHelper.SMembers(keyname.Replace(":total:", ":coupon_total:"));
            //message_keys = message_keys.Union(message_keys3).ToArray();
        }


        //if (item.total_count > 0) continue;
        //item.total_count = RedisHelper.Get<int>($":total:all:{item.report_date:yyyyMMdd}");
        //if (item.total_count == 0) continue;
        //item.success_count = RedisHelper.Get<int>($":total:all:success:{item.report_date:yyyyMMdd}");
        //item.abandon_count = RedisHelper.Get<int>($":total:all:放弃转链:{item.report_date:yyyyMMdd}");




    }

}