|
|
@@ -158,26 +158,30 @@ namespace molilian.core
|
|
|
return Regex.IsMatch(content, pattern);
|
|
|
}
|
|
|
|
|
|
- public TimeSpan GetRequestTimeout(int deduction = 0)
|
|
|
+ public TimeSpan GetRequestTimeout()
|
|
|
{
|
|
|
-#if DEBUG
|
|
|
- return TimeSpan.FromMilliseconds(10 * 1000);
|
|
|
-#endif
|
|
|
- int min = (int)(_config.rt_max * 0.3);
|
|
|
+ int min = (int)(_config.rt_max * 0.5);
|
|
|
+ if (_config.rt_max == 0) return TimeSpan.FromMilliseconds(1000);
|
|
|
+ int timeout = _config.rt_max;
|
|
|
+
|
|
|
+ timeout -= min;
|
|
|
+ return TimeSpan.FromMilliseconds(timeout);
|
|
|
+
|
|
|
+ }
|
|
|
+
|
|
|
+ public TimeSpan GetRequestTimeout2(int deduction = 0)
|
|
|
+ {
|
|
|
+ int min = (int)(_config.rt_max * 0.5);
|
|
|
if (_config.rt_max == 0) return TimeSpan.FromMilliseconds(1000);
|
|
|
|
|
|
int timeout = _config.rt_max;
|
|
|
- if (deduction == 0)
|
|
|
- {
|
|
|
- timeout -= min;
|
|
|
- return TimeSpan.FromMilliseconds(timeout);
|
|
|
- }
|
|
|
|
|
|
timeout -= deduction;
|
|
|
- if (timeout <= min) return TimeSpan.FromMilliseconds(1);
|
|
|
- return TimeSpan.FromMilliseconds(min);
|
|
|
+ if (timeout <= min) return TimeSpan.FromMilliseconds(min);
|
|
|
+ return TimeSpan.FromMilliseconds(timeout);
|
|
|
}
|
|
|
|
|
|
+
|
|
|
/// <summary>
|
|
|
/// 从http获取跳转的url
|
|
|
/// </summary>
|
|
|
@@ -212,7 +216,7 @@ namespace molilian.core
|
|
|
client.Proxy = null;
|
|
|
var response = await client.RequestAsync(url, "GET");
|
|
|
#else
|
|
|
- client.Timeout = GetRequestTimeout(0);
|
|
|
+ client.Timeout = GetRequestTimeout();
|
|
|
var response = await client.RequestAsync(url, "GET", cancellationToken);
|
|
|
#endif
|
|
|
|
|
|
@@ -281,7 +285,7 @@ namespace molilian.core
|
|
|
#if DEBUG
|
|
|
client.Proxy = null;
|
|
|
#endif
|
|
|
- client.Timeout = GetRequestTimeout(0);
|
|
|
+ client.Timeout = GetRequestTimeout();
|
|
|
var response = client.Request(url);
|
|
|
|
|
|
|
|
|
@@ -614,6 +618,290 @@ namespace molilian.core
|
|
|
return LinkTypeEnum.unknown;
|
|
|
}
|
|
|
|
|
|
+ public async Task<TkDataDTO> alimamaParseAsync2(string content, TkDataDTO result, CancellationToken cancellationToken = default)
|
|
|
+ {
|
|
|
+ string t = $"{DateTime.Now.Convert2UnixTimestamp(true)}";
|
|
|
+
|
|
|
+ Random random = new();
|
|
|
+ double randomNumber = random.NextDouble();
|
|
|
+ string randomString = randomNumber.ToString()[2..];
|
|
|
+
|
|
|
+ string cna = _cookies.GetContentPart("cna=", ";");
|
|
|
+ string firstFiveCharacters = cna[..Math.Min(5, cna.Length)];
|
|
|
+
|
|
|
+ //var variableMap = new
|
|
|
+ //{
|
|
|
+ // url = content,
|
|
|
+ // union_lens = $"b_pvid:a219t._portal_v2_tool_links_page_home_index_htm_{t}_{randomString}_{firstFiveCharacters}",
|
|
|
+ // lensScene = "PUB",
|
|
|
+ // spmB = "_portal_v2_tool_links_page_home_index_htm"
|
|
|
+ //}.Convert2Json(true).UrlEncode();
|
|
|
+
|
|
|
+ string item_id = content.GetContentPart("item.htm?id=", "");
|
|
|
+ var variableMap = new
|
|
|
+ {
|
|
|
+ itemId = item_id
|
|
|
+ }.Convert2Json(true).UrlEncode();
|
|
|
+
|
|
|
+
|
|
|
+ variableMap = variableMap.Replace(" ", "%20");
|
|
|
+ _floorId = "102359";
|
|
|
+ string url = "https://pub.alimama.com/openapi/param2/1/gateway.unionpub/xt.entry.json?" +
|
|
|
+ $"t={t}&_tb_token_={_tb_token}&floorId={_floorId}&refpid={_refpid}&variableMap={variableMap}";
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+ Stopwatch stopwatch = Stopwatch.StartNew();
|
|
|
+ stopwatch.Start();
|
|
|
+ var client = new WebClientUtility().SetContentType("application/json;charset=utf-8")
|
|
|
+ .AddHeaders("X-Requested-With", "XMLHttpRequest")
|
|
|
+ .AddHeaders("Cookie", _cookies);
|
|
|
+ client.Proxy = _proxy;
|
|
|
+#if DEBUG
|
|
|
+ client.Proxy = null;
|
|
|
+#endif
|
|
|
+ if (!string.IsNullOrEmpty(_user_agent)) client.UserAgent = _user_agent;
|
|
|
+ client.Timeout = GetRequestTimeout2(result.elapsedTime2);
|
|
|
+ //var response = client.Request(url);
|
|
|
+
|
|
|
+#if DEBUG
|
|
|
+ var response = await client.RequestAsync(url, "GET");
|
|
|
+#else
|
|
|
+ var response = await client.RequestAsync(url, "GET", cancellationToken);
|
|
|
+#endif
|
|
|
+
|
|
|
+ stopwatch.Stop();
|
|
|
+ result.elapsedTime3 = (int)stopwatch.ElapsedMilliseconds;
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+ var body = string.Empty;
|
|
|
+ try
|
|
|
+ {
|
|
|
+ if (!response.Successed)
|
|
|
+ {
|
|
|
+ result.channel = TkChannelEnum.tb;
|
|
|
+ result.link_type = LinkTypeEnum.unknown;
|
|
|
+ result.success = false;
|
|
|
+ result.message = "fail";
|
|
|
+
|
|
|
+ if (response.ResponseException != null)
|
|
|
+ {
|
|
|
+ _ = new LoggerLibrary("api_error", "fail")
|
|
|
+ .Info(response.ResponseException.Message, response.ResponseException.StackTrace)
|
|
|
+ .SaveAsync();
|
|
|
+ throw response.ResponseException;
|
|
|
+ }
|
|
|
+
|
|
|
+ return result;
|
|
|
+ }
|
|
|
+ if (response.ResponseMessage.StatusCode == System.Net.HttpStatusCode.Found)
|
|
|
+ {
|
|
|
+ _ = new LoggerLibrary("api_error", "fail")
|
|
|
+ .Info($"302:{response.ResponseMessage.Headers.Location}")
|
|
|
+ .SaveAsync();
|
|
|
+ throw new Exception("302 Found, was canceled");
|
|
|
+ }
|
|
|
+ JsonElement root;
|
|
|
+ try
|
|
|
+ {
|
|
|
+ body = response.Body();
|
|
|
+ //{"success":false,"message":"result empty"}
|
|
|
+
|
|
|
+
|
|
|
+ root = body.Convert2JsonElement();
|
|
|
+ }
|
|
|
+ catch (Exception ex)
|
|
|
+ {
|
|
|
+ _ = new LoggerLibrary("api_error", "fail")
|
|
|
+ .Info(ex.Message, ex.StackTrace)
|
|
|
+ .SaveAsync();
|
|
|
+
|
|
|
+ if (body.Contains("https://g.alicdn.com/sd/punish/waf_block.html"))
|
|
|
+ {
|
|
|
+
|
|
|
+ result.channel = TkChannelEnum.tb;
|
|
|
+ result.link_type = LinkTypeEnum.unknown;
|
|
|
+ result.success = false;
|
|
|
+ result.message = "waf_block";
|
|
|
+ result.reason = "接口风控";
|
|
|
+ return result;
|
|
|
+ }
|
|
|
+ throw new Exception(body);
|
|
|
+ }
|
|
|
+
|
|
|
+
|
|
|
+ bool success = root.Read<bool>("success", false);
|
|
|
+ string message = root.Read("message", string.Empty);
|
|
|
+ string info_message = root.PathRead("info.message", string.Empty);
|
|
|
+ if (!string.IsNullOrEmpty(info_message)) message = info_message;
|
|
|
+
|
|
|
+ if ("result empty".Equals(message))
|
|
|
+ {
|
|
|
+ result.subCode = TkSubCodeEnum.NoConvert;
|
|
|
+
|
|
|
+ }
|
|
|
+ JsonElement item = default;
|
|
|
+
|
|
|
+ if (success)
|
|
|
+ {
|
|
|
+ var data = root.ElementRead("data");
|
|
|
+ var resultList = data.ElementRead("resultList");
|
|
|
+ if (resultList.ValueKind != JsonValueKind.Array)
|
|
|
+ {
|
|
|
+ _ = new LoggerLibrary("api_error", "fail")
|
|
|
+ .Info("resultList Undefined", body)
|
|
|
+ .SaveAsync();
|
|
|
+ throw new Exception("resultList Undefined");
|
|
|
+ }
|
|
|
+ item = resultList[0];
|
|
|
+ }
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+ string taoToken = item.Read("taoToken", string.Empty);
|
|
|
+ string shortLinkurl = item.Read("shortLinkurl", string.Empty);
|
|
|
+
|
|
|
+ string couponLinkTaoToken = item.Read("couponLinkTaoToken", string.Empty);
|
|
|
+ string couponShortLinkUrl = item.Read("couponShortLinkUrl", string.Empty);
|
|
|
+ decimal couponAmount = item.Read<decimal>("couponAmount", 0);
|
|
|
+ string couponEffectiveEndTime = item.Read("couponEffectiveEndTime", string.Empty);
|
|
|
+ string couponEffectiveStartTime = item.Read("couponEffectiveStartTime", string.Empty);
|
|
|
+
|
|
|
+ string itemId = item.Read("itemId", string.Empty);
|
|
|
+ string mktId = item.Read("outputMktId", string.Empty);
|
|
|
+ string itemName = item.Read("itemName", string.Empty);
|
|
|
+ decimal promotionPrice = item.Read<decimal>("promotionPrice", 0);
|
|
|
+ string sellerNickName = item.Read("sellerNickName", string.Empty);
|
|
|
+ string shopTitle = item.Read("shopTitle", string.Empty);
|
|
|
+ string pic = item.Read("data.pic", string.Empty);
|
|
|
+ if (pic.StartsWith("//")) pic = "https:" + pic;
|
|
|
+
|
|
|
+ string qrCodeUrl = item.Read("qrCodeUrl", string.Empty);
|
|
|
+
|
|
|
+
|
|
|
+ if (response.ResponseMessage != null)
|
|
|
+ {
|
|
|
+
|
|
|
+ switch (response.ResponseMessage.StatusCode)
|
|
|
+ {
|
|
|
+ case System.Net.HttpStatusCode.OK:
|
|
|
+ {
|
|
|
+ if (couponAmount > 0 && !string.IsNullOrEmpty(couponLinkTaoToken) && !string.IsNullOrEmpty(couponShortLinkUrl))
|
|
|
+ {
|
|
|
+ taoToken = couponLinkTaoToken;
|
|
|
+ shortLinkurl = couponShortLinkUrl;
|
|
|
+ }
|
|
|
+ string shortLink = GetLink(taoToken);
|
|
|
+ content = ReplaceUrls(content, shortLink);
|
|
|
+ }
|
|
|
+ break;
|
|
|
+ case System.Net.HttpStatusCode.Found:
|
|
|
+ string location = response.ResponseMessage.Headers.Location.OriginalString;
|
|
|
+ if (!string.IsNullOrEmpty(location) && location.Contains("www.alimama.com/member/login.htm"))
|
|
|
+ {
|
|
|
+ success = false;
|
|
|
+ message = "nologin";
|
|
|
+ }
|
|
|
+ break;
|
|
|
+ default:
|
|
|
+ {
|
|
|
+ success = false;
|
|
|
+ message = "other";
|
|
|
+ }
|
|
|
+ break;
|
|
|
+ }
|
|
|
+ }
|
|
|
+ if (!success)
|
|
|
+ {
|
|
|
+ shortLinkurl = GetLink(content);
|
|
|
+ switch (message)
|
|
|
+ {
|
|
|
+ case "该商品已经下架或未加入淘宝客":
|
|
|
+ case "该链接不支持转化,请更换链接尝试":
|
|
|
+ case "result empty":
|
|
|
+ result.subCode = TkSubCodeEnum.NoConvert;
|
|
|
+ break;
|
|
|
+ case "网络错误":
|
|
|
+ result.subCode = TkSubCodeEnum.NetError;
|
|
|
+ break;
|
|
|
+ default:
|
|
|
+ var _headers = response.ResponseMessage.Headers;
|
|
|
+ var headers = "";
|
|
|
+ foreach (var h in _headers)
|
|
|
+ {
|
|
|
+ foreach (var Value in h.Value)
|
|
|
+ {
|
|
|
+ headers += $"{h.Key}: {Value}\n";
|
|
|
+ }
|
|
|
+ }
|
|
|
+ _ = new LoggerLibrary("api_error", "fail")
|
|
|
+ .Info(message, content)
|
|
|
+ .Info(headers, body)
|
|
|
+ .SaveAsync();
|
|
|
+ break;
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ shortLinkurl = GetLink(taoToken);
|
|
|
+ string deeplink_url = GetDeeplink(shortLinkurl);
|
|
|
+
|
|
|
+ result.channel = TkChannelEnum.tb;
|
|
|
+
|
|
|
+ if (success)
|
|
|
+ {
|
|
|
+ result.subCode = TkSubCodeEnum.Success;
|
|
|
+ switch (result.link_type)
|
|
|
+ {
|
|
|
+ case LinkTypeEnum.profile:
|
|
|
+ case LinkTypeEnum.goods:
|
|
|
+ case LinkTypeEnum.video:
|
|
|
+ case LinkTypeEnum.live:
|
|
|
+ break;
|
|
|
+ default:
|
|
|
+ result.link_type = LinkTypeEnum.goods;
|
|
|
+ break;
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ result.success = success;
|
|
|
+ result.message = message;
|
|
|
+ result.reason = string.Empty;
|
|
|
+
|
|
|
+ result.content = content;
|
|
|
+ result.couponAmount = couponAmount;
|
|
|
+ result.couponEffectiveEndTime = couponEffectiveEndTime;
|
|
|
+ result.couponEffectiveStartTime = couponEffectiveStartTime;
|
|
|
+ result.taoToken = taoToken;
|
|
|
+ if (!string.IsNullOrEmpty(shortLinkurl))
|
|
|
+ {
|
|
|
+ result.shortLinkurl = shortLinkurl;
|
|
|
+ result.deeplink_url = deeplink_url;
|
|
|
+ }
|
|
|
+ result.mktId = mktId;
|
|
|
+
|
|
|
+ result.itemId = itemId;
|
|
|
+ result.itemName = itemName;
|
|
|
+ result.promotionPrice = promotionPrice;
|
|
|
+ result.sellerNickName = sellerNickName;
|
|
|
+ result.shopTitle = shopTitle;
|
|
|
+ result.pic = pic;
|
|
|
+ result.qrCodeUrl = qrCodeUrl;
|
|
|
+ }
|
|
|
+ catch (Exception ex)
|
|
|
+ {
|
|
|
+ if (response.ResponseMessage?.StatusCode == System.Net.HttpStatusCode.OK)
|
|
|
+ {
|
|
|
+ if (body.Contains("<p>抱歉!页面无法访问……</p>"))
|
|
|
+ {
|
|
|
+ throw new APIException("抱歉!页面无法访问……");
|
|
|
+ }
|
|
|
+ }
|
|
|
+ throw;
|
|
|
+ }
|
|
|
+ return result;
|
|
|
+ }
|
|
|
+
|
|
|
public async Task<TkDataDTO> alimamaParseAsync(string content, TkDataDTO result, CancellationToken cancellationToken = default)
|
|
|
{
|
|
|
string t = $"{DateTime.Now.Convert2UnixTimestamp(true)}";
|
|
|
@@ -650,7 +938,7 @@ namespace molilian.core
|
|
|
client.Proxy = null;
|
|
|
#endif
|
|
|
if (!string.IsNullOrEmpty(_user_agent)) client.UserAgent = _user_agent;
|
|
|
- client.Timeout = GetRequestTimeout(result.elapsedTime2);
|
|
|
+ client.Timeout = GetRequestTimeout2(result.elapsedTime2);
|
|
|
//var response = client.Request(url);
|
|
|
|
|
|
#if DEBUG
|
|
|
@@ -712,6 +1000,17 @@ namespace molilian.core
|
|
|
result.message = "waf_block";
|
|
|
return result;
|
|
|
}
|
|
|
+
|
|
|
+ if (body.Contains("https://bixi.alicdn.com") && body.Contains("\"action\": \"deny\""))
|
|
|
+ {
|
|
|
+ result.channel = TkChannelEnum.tb;
|
|
|
+ result.link_type = LinkTypeEnum.unknown;
|
|
|
+ result.subCode = TkSubCodeEnum.ParseDeny;
|
|
|
+ result.success = false;
|
|
|
+ result.message = "waf_block";
|
|
|
+ result.reason = "接口风控";
|
|
|
+ return result;
|
|
|
+ }
|
|
|
throw new Exception(body);
|
|
|
}
|
|
|
|
|
|
@@ -902,7 +1201,7 @@ namespace molilian.core
|
|
|
$"t={t}&_tb_token_={_tb_token}&floorId={_floorId}&refpid={_refpid}&variableMap={variableMap}";
|
|
|
client.Headers["Host"] = "pub.alimama.com";
|
|
|
#endif
|
|
|
- client.Timeout = GetRequestTimeout(result.elapsedTime2);
|
|
|
+ client.Timeout = GetRequestTimeout2(result.elapsedTime2);
|
|
|
//var response = client.Request(url);
|
|
|
var response = client.Request(url);
|
|
|
|