dotnet/molilian.core/Plus/Alimama/parse_2.cs

using dodohold.core;
using System.Text.RegularExpressions;
using System.Diagnostics;
using System.Text.Json;
using TencentCloud.Fmu.V20191213.Models;
using System;
using MySqlX.XDevAPI;
using static dodohold.core.ZTOExpress.CreateOrderArgs;
using System.Threading;
using TencentCloud.Tcm.V20210413.Models;
using COSXML.Network;
using System.Net;
using TencentCloud.Cdb.V20170320.Models;
using System.Security.Cryptography;

namespace molilian.core
{
    public partial class AlimamaPlus
    {
        public static async Task<TkDataDTO> testTaskAsync(TkDataDTO result, CancellationToken token)
        {
            string url = "https://www.taobao.com";
            try
            {
                WebClientUtility client = new WebClientUtility();
                await Task.Delay(5000);
                client.Timeout = TimeSpan.FromMilliseconds(3000);
                var response = await client.RequestAsync(url);
                var body = response.Body();
            }
            catch
            {
                result.success = false;
                result.message = "Error";
            }
            return result;
        }

        /// <summary>
        /// 提取url里面的商品id参数
        /// </summary>
        /// <param name="url"></param>
        /// <returns></returns>
        public static string? ExtractItemId(string url)
        {
            Match match = Regex.Match(url, @"(?:\?|&)(?:id|shareDetailItemId|itemIds)=(\d+)");
            if (match.Success)
            {
                return match.Groups[1].Value;
            }
            return null;
        }

        /// <summary>
        /// 是否淘宝链接，如果是商品链接就格式化成https://item.taobao.com/item.htm?id=****
        /// </summary>
        /// <param name="url"></param>
        /// <returns></returns>
        public static (bool, string) IsTaobaoUrl(string url)
        {
            if (url.Contains("//a.m.taobao.com/i") ||
                url.Contains("//internal.tt.detail.taobao.com/i"))
            {
                //第一种提取方法
                string itemId = url.GetContentPart(".com/i", ".htm");
                if (!string.IsNullOrEmpty(itemId) && long.TryParse(itemId, out long _))
                {
                    string resultUrl = $"https://item.taobao.com/item.htm?id={itemId}";
                    return (true, resultUrl);
                }
            }
            //https://pages-g.m.taobao.com/wow/z/app/detail-next/item/index?x-ssr=true&id=753169018016

            if (url.Contains("//item.taobao.com/item.htm?") ||
                url.Contains("//pages-g.m.taobao.com/wow/z/app/detail-next/item/index?") ||
                (url.Contains("//pages-fast.m.taobao.com/wow/bz/jingmi/") && url.Contains("shareDetailItemId=")) ||
                url.Contains("//main.m.taobao.com/detail/index.html?") ||
                url.Contains("//main.m.taobao.com/security-h5-detail/home?") ||
                url.Contains("//h5.m.taobao.com/awp/core/detail.htm?") ||
                url.Contains("//new.m.taobao.com/detail.htm?") ||
                url.Contains("//outfliggys.m.taobao.com/app/trip/rx-travel-detail") ||
                url.Contains("//detail.m.tmall.com/item.htm?") ||
                url.Contains("//tmallx.tmall.com/app/tmallx-src/goods-detail") ||
                url.Contains("//detail.tmall.com/item.htm?") ||
                url.Contains("//internal.tt.detail.taobao.com/item.htm?") ||
                url.Contains("//internal.tt.detail.taobao.com/") ||
                url.Contains("//detail.tmall.hk/hk/item.htm?"))
            {
                string? itemId = ExtractItemId(url);
                if (!string.IsNullOrEmpty(itemId))
                {
                    string resultUrl = $"https://item.taobao.com/item.htm?id={itemId}";
                    return (true, resultUrl);
                }
            }

            return (false, url);
        }

        /// <summary>
        /// 排除已知的淘客链接
        /// </summary>
        /// <param name="url"></param>
        /// <returns></returns>
        public static bool IsAffLink(string url)
        {
            if (!url.StartsWith("https://") || !url.StartsWith("https://")) url = "https://" + url;
            Uri uri = new(url);
            if (uri.Host.Equals("s.click.taobao.com", StringComparison.OrdinalIgnoreCase) ||
                uri.Host.Equals("uland.taobao.com", StringComparison.OrdinalIgnoreCase))
            {
                return true;
            }
            return false;
        }
        public static bool IsShortLink(string url)
        {
            if (!url.StartsWith("https://", StringComparison.OrdinalIgnoreCase) &&
                !url.StartsWith("http://", StringComparison.OrdinalIgnoreCase)) url = "https://" + url;
            Uri uri = new(url);
            if (uri.Host.Equals("e.tb.cn", StringComparison.OrdinalIgnoreCase) ||
                uri.Host.Equals("m.tb.cn", StringComparison.OrdinalIgnoreCase) ||
                uri.Host.Equals("u.tb.cn", StringComparison.OrdinalIgnoreCase) ||
                uri.Host.Equals("s.tb.cn", StringComparison.OrdinalIgnoreCase))
            {
                return true;
            }
            return false;
        }

        /// <summary>
        /// 判断淘客的￥羊角符号
        /// </summary>
        /// <param name="content"></param>
        /// <returns></returns>
        static bool ContainsSpecialFormat(string content)
        {
            // 使用正则表达式匹配内容中是否含有特定格式的字符串
            //string pattern = @"￥[a-zA-Z0-9\s]+￥";
            string pattern = @"[$￥🗝₴€₤£₳✔《💰🔑🎵✔️💲🔐📲₡]+[a-zA-Z0-9\s]+[$￥🗝₴€₤£₳✔《💰🔑🎵✔️💲🔐📲₡]*";
            return Regex.IsMatch(content, pattern);
        }

        public TimeSpan GetRequestTimeout()
        {
            int min = (int)(_config.rt_max * 0.5);
            if (_config.rt_max == 0) return TimeSpan.FromMilliseconds(1000);
            int timeout = _config.rt_max;

            timeout -= min;
            return TimeSpan.FromMilliseconds(timeout);

        }

        public TimeSpan GetRequestTimeout2(int deduction = 0)
        {
            int min = (int)(_config.rt_max * 0.5);
            if (_config.rt_max == 0) return TimeSpan.FromMilliseconds(1000);

            int timeout = _config.rt_max;

            timeout -= deduction;
            if (timeout <= min) return TimeSpan.FromMilliseconds(min);
            return TimeSpan.FromMilliseconds(timeout);
        }


        /// <summary>
        /// 从http获取跳转的url
        /// </summary>
        /// <param name="url"></param>
        /// <returns></returns>
        public async Task<(bool, string)> GetDesiredUrlAsync(WebProxy proxy, string url, CancellationToken cancellationToken = default)
        {
            string result;
            try
            {
                if (!IsShortLink(url)) return (false, "不是淘宝短网址");

                if (!_account.enable_deep_url) return (true, url);

                string desiredUrlPattern = "var url = '(.*?)'";
                WebClientUtility client = new()
                {
                    Proxy = proxy,
                    UserAgent = Sayaka.Common.ProviderFakeUserAgent.RandomComputer
                };

                if (url.Contains("u.tb.cn"))
                {
                    client.AllowAutoRedirect = false;
                }
                url = url.Replace("http://u.tb.cn", "https://u.tb.cn");
#if DEBUG
                client.Proxy = null;
                var response = await client.RequestAsync(url, "GET");
#else
                client.Timeout = GetRequestTimeout();
                var response = await client.RequestAsync(url, "GET", cancellationToken);
#endif

                if (response.ResponseMessage?.StatusCode == HttpStatusCode.Found &&
                    !string.IsNullOrEmpty(response.ResponseMessage?.Headers?.Location?.OriginalString))
                {
                    string redirectUrl = response.ResponseMessage?.Headers?.Location?.OriginalString;
                    return (true, redirectUrl);
                }
                var responseBody = response.Body();
                result = responseBody;
                if (response.Successed)
                {

                    if (response.ResponseMessage.IsSuccessStatusCode)
                    {
                        Match match = Regex.Match(responseBody, desiredUrlPattern);
                        if (match.Success)
                        {
                            result = match.Groups[1].Value; // 返回匹配的 URL

                            if (string.IsNullOrEmpty(result)) return (false, "没有匹配的URL");
                            if (result.StartsWith("//"))
                                result = "https:" + result;
                            return (true, result);
                        }
                    }
                    else
                    {
                        result = $"{response.ResponseMessage.StatusCode}\n{responseBody}";
                    }
                }
                else
                {
                    throw response.ResponseException;

                }
            }
            catch (Exception ex)
            {
                _ = new LoggerLibrary("api_error", "desiredUrl_error").Info(ex.Message, ex.StackTrace).SaveAsync();
                result = "网络异常";
            }
            return (false, result);
        }

        /// <summary>
        /// 从http获取跳转的url
        /// </summary>
        /// <param name="url"></param>
        /// <returns></returns>
        public (bool, string) GetDesiredUrl(string url)
        {
            string result;
            try
            {
                if (!IsShortLink(url)) return (false, "不是淘宝短网址");

                string desiredUrlPattern = "var url = '(.*?)'";
                WebClientUtility client = new()
                {
                    Proxy = ProxyNodesCore.RandomOne(_account.nodeName),
                    UserAgent = Sayaka.Common.ProviderFakeUserAgent.RandomComputer
                };
#if DEBUG
                client.Proxy = null;
#endif
                client.Timeout = GetRequestTimeout();
                var response = client.Request(url);


                var responseBody = response.Body();
                result = responseBody;
                if (response.Successed)
                {

                    if (response.ResponseMessage.IsSuccessStatusCode)
                    {
                        Match match = Regex.Match(responseBody, desiredUrlPattern);
                        if (match.Success)
                        {
                            result = match.Groups[1].Value; // 返回匹配的 URL

                            if (string.IsNullOrEmpty(result)) return (false, "没有匹配的URL");
                            return (true, result);
                        }
                    }
                    else
                    {
                        result = $"{response.ResponseMessage.StatusCode}\n{responseBody}";
                    }
                }
                else
                {
                    throw response.ResponseException;

                }
            }
            catch (Exception ex)
            {
                result = $"{ex.Message}\n{ex.StackTrace}";
            }
            return (false, result);
        }

        public static (bool, string) GetItemUrl(string url, WebProxy proxy = null)
        {
            string result;
            try
            {
                if (!IsShortLink(url)) return (false, "不是淘宝短网址");

                string desiredUrlPattern = "var url = '(.*?)'";
                WebClientUtility client = new()
                {
                    Proxy = proxy,
                    UserAgent = Sayaka.Common.ProviderFakeUserAgent.RandomComputer
                };
#if DEBUG
                client.Proxy = null;
#endif
                var response = client.Request(url);

                var responseBody = response.Body();
                result = responseBody;
                if (response.Successed)
                {

                    if (response.ResponseMessage.IsSuccessStatusCode)
                    {
                        Match match = Regex.Match(responseBody, desiredUrlPattern);
                        if (match.Success)
                        {
                            result = match.Groups[1].Value; // 返回匹配的 URL

                            if (string.IsNullOrEmpty(result)) return (false, "没有匹配的URL");
                            return (true, result);
                        }
                    }
                    else
                    {
                        result = $"{response.ResponseMessage.StatusCode}\n{responseBody}";
                    }
                }
                else
                {
                    throw response.ResponseException;

                }
            }
            catch (Exception ex)
            {
                result = $"{ex.Message}\n{ex.StackTrace}";
            }
            return (false, result);
        }


        private void InternalTextProcessing(string content, ref TkDataDTO result)
        {
            //(result.success, result.content, result.link_type, result.shortLinkurl) = InternalTextProcessing(content,ref result);


            bool success;
            string resultText;
            LinkTypeEnum link_type = LinkTypeEnum.unknown;

            string url = GetLink(content);
            if (!string.IsNullOrEmpty(url))
            {
                //排除已知的淘客链接
                if (IsAffLink(url))
                {
                    result.success = false;
                    result.content = "排除淘客链接";
                    result.link_type = LinkTypeEnum.other_aff;
                    result.shortLinkurl = url;
                    return;
                    //return (false, "排除淘客链接", LinkTypeEnum.other_aff, url);
                }

                //直接提取
                (success, resultText) = IsTaobaoUrl(url);
                if (success)
                {
                    result.success = true;
                    result.content = resultText;
                    result.link_type = LinkTypeEnum.goods;
                    result.shortLinkurl = resultText;
                    return;
                    //return (true, result, LinkTypeEnum.goods, result);
                }

                //从http获取跳转的url
                Stopwatch stopwatch = Stopwatch.StartNew();
                stopwatch.Start();
                (success, resultText) = GetDesiredUrl(url);
                stopwatch.Stop();
                // 获取执行时间
                result.elapsedTime2 = (int)stopwatch.ElapsedMilliseconds;

                if (success)
                {
                    //排除已知的淘客链接
                    if (IsAffLink(resultText))
                    {
                        result.success = false;
                        result.content = "排除淘客链接";
                        result.link_type = LinkTypeEnum.other_aff;
                        result.shortLinkurl = url;
                        return;
                        //return (false, "排除淘客链接", LinkTypeEnum.other_aff, url);
                    }
                    (success, resultText) = IsTaobaoUrl(resultText);
                    if (success)
                    {
                        result.success = true;
                        result.content = resultText;
                        result.link_type = LinkTypeEnum.goods;
                        result.shortLinkurl = resultText;
                        return;
                        //return (true, resultText, LinkTypeEnum.goods, resultText);
                    }
                    link_type = GetLinkType(resultText);
                }
                else
                {
                    link_type = GetLinkType(url);

                }

                result.success = false;
                result.content = resultText;
                result.link_type = link_type;
                result.shortLinkurl = url;
                return;

                //return (false, result, link_type, url);
            }

            //判断淘客的￥羊角符号
            if (ContainsSpecialFormat(content))
            {
                result.success = false;
                result.content = "排除淘口令";
                result.link_type = LinkTypeEnum.other_aff;
                result.shortLinkurl = null;
                return;
                //return (false, "排除淘口令", LinkTypeEnum.other_aff, null);
            }

            //todo 临时应急  0614
            //return (false, "纯口令 没链接", link_type, null);
            result.success = false;
            result.content = "纯口令 没链接";
            result.link_type = LinkTypeEnum.other_aff;
            result.shortLinkurl = null;
        }

        public async Task<TkDataDTO> InternalTextProcessingAsync(string content, TkDataDTO result,
            CancellationToken cancellationToken = default,
            Dictionary<string, long> swData = null,
            bool preserveShortLinks = false)
        {
            bool success;
            string resultText;
            result.link_type = LinkTypeEnum.unknown;

            //============================== 放弃转链-其他推广链接 ==============================
            string url = GetLink(content);
            result.shortLinkurl = url;

            if (!string.IsNullOrEmpty(url))
            {
                bool is_aff = IsAffLink(url);
                if (is_aff)
                {
                    result.success = false;
                    result.message = "放弃转链";
                    result.reason = "其他推广链接";
                    result.subCode = TkSubCodeEnum.OtherPromo;
                    result.link_type = LinkTypeEnum.other_aff;
                    //result.shortLinkurl = url;
                    return result;
                }

                (success, resultText) = IsTaobaoUrl(url);
                if (success)
                {
                    result.success = true;
                    result.content = resultText;
                    result.link_type = LinkTypeEnum.goods;
                    //result.shortLinkurl = resultText;
                    return result;
                }


                if (preserveShortLinks && IsShortLink(url))
                {
                    result.success = false;
                    result.message = string.Empty;
                    result.reason = string.Empty;
                    result.content = content;
                    result.link_type = LinkTypeEnum.baseDomain;
                    result.elapsedTime2 = 0;
                    return result;
                }

                Stopwatch sw = Stopwatch.StartNew();

                var proxy = ProxyNodesCore.RandomOne(_account.nodeName);
                if (_proxy != null && proxy != null && _proxy.Address != proxy.Address)
                {
                    result.proxy_node = proxy?.Address?.Authority;
                }

                (success, resultText) = await GetDesiredUrlAsync(proxy, url, cancellationToken);
                sw.Stop();
                // 获取执行时间
                result.elapsedTime2 = (int)sw.ElapsedMilliseconds;
                swData?.Add("\tGetDesiredUrlAsync", sw.ElapsedMilliseconds);
                //============================== 放弃转链-请求超时 ==============================
#if DEBUG
#else
                if (result.elapsedTime2 >= _config.rt_max || (!success && resultText.Contains("was canceled")))
                {
                    result.success = false;
                    result.message = "放弃转链";
                    result.reason = "请求超时";
                    result.subCode = TkSubCodeEnum.Other;
                    return result;
                }
#endif
                if (!success && resultText.Contains("<!DOCTYPE html>"))
                {
                    result.success = false;
                    result.message = "放弃转链";
                    result.reason = "短链解析错误";
                    result.subCode = TkSubCodeEnum.Other;
                    return result;
                }


                if (success)
                {
                    string desiredUrl = resultText;

                    //============================== 放弃转链-其他推广链接 ==============================
                    bool is_aff2 = IsAffLink(desiredUrl);
                    if (is_aff2)
                    {
                        result.success = false;
                        result.message = "放弃转链";
                        result.reason = "其他推广链接";
                        result.subCode = TkSubCodeEnum.OtherPromo;
                        result.link_type = LinkTypeEnum.other_aff;
                        //result.shortLinkurl = url;
                        return result;
                    }

                    (success, desiredUrl) = IsTaobaoUrl(desiredUrl);
                    if (success)
                    {
                        result.success = true;
                        result.content = desiredUrl;
                        result.link_type = LinkTypeEnum.goods;
                        //result.shortLinkurl = desiredUrl;
                        return result;
                    }
                    result.link_type = GetLinkType(desiredUrl);
                }
                else
                {
                    result.link_type = GetLinkType(url);
                    if ("网络错误".Equals(resultText))
                    {
                        result.link_type = LinkTypeEnum.unknown;
                        result.content = resultText;
                        result.reason = resultText;
                    }
                }

                if (result.content.Contains("霸下通用 web 页面-验证码"))
                {
                    result.subCode = TkSubCodeEnum.Captcha;
                    result.reason = "霸下验证码";
                }
                else
                {
                    switch (result.link_type)
                    {
                        case LinkTypeEnum.profile:
                        case LinkTypeEnum.live:
                        case LinkTypeEnum.video:
                        case LinkTypeEnum.note:
                        case LinkTypeEnum.goods:
                        case LinkTypeEnum.baseDomain:
                            result.reason = string.Empty;
                            result.content = content;
                            break;

                        default:
                            result.reason = "非标准链接";
                            result.subCode = TkSubCodeEnum.NonStdLink;
                            break;
                    }

                    //if (result.link_type == LinkTypeEnum.baseDomain)
                    //{
                    //    result.content = content;
                    //}
                    //else
                    //{
                    //    result.reason = "非标准链接";
                    //    result.subCode = TkSubCodeEnum.NonStdLink;

                    //}
                }
                result.success = false;
                result.message = "放弃转链";
                result.content = resultText;
                result.shortLinkurl = url;

                return result;
                //return (false, result, link_type, url);
            }


            //判断淘客的￥羊角符号
            bool is_aff3 = ContainsSpecialFormat(content);
            if (is_aff3)
            {
                result.success = false;
                result.message = "放弃转链";
                result.reason = "排除淘口令";
                result.subCode = TkSubCodeEnum.OtherPromo;
                result.link_type = LinkTypeEnum.other_aff;
                result.shortLinkurl = null;
                return result;
                //return (false, "排除淘口令", LinkTypeEnum.other_aff, null);
            }

            //todo 临时应急  0614
            //return (false, "纯口令 没链接", link_type, null);
            result.success = false;
            result.message = "放弃转链";
            result.reason = "纯口令 没链接";
            result.subCode = TkSubCodeEnum.OtherPromo;
            result.link_type = LinkTypeEnum.other_aff;
            result.shortLinkurl = null;
            return result;
        }

        private LinkTypeEnum GetLinkType(string url)
        {
            if (string.IsNullOrEmpty(url)) return LinkTypeEnum.unknown;

            if (IsShortLink(url)) return LinkTypeEnum.baseDomain;

            if (url.StartsWith("https://huodong.m.taobao.com/act/talent/live.html")) return LinkTypeEnum.live;
            if (url.StartsWith("https://web.m.taobao.com/app/tnode/web/index")) return LinkTypeEnum.video;
            if (url.StartsWith("https://shop.m.taobao.com/shop/shopIndex.htm")) return LinkTypeEnum.profile;

            string pattern = @"^https://shop\d+(\.m)*\.taobao\.com";
            if (Regex.IsMatch(url, pattern))
            {
                return LinkTypeEnum.profile;
            }
            //https://shop.m.taobao.com/shop/shopIndex.htm?

            return LinkTypeEnum.unknown;
        }


    }
}