Skip to content

D
data-center-auto
Commit 0328f60a by lizefeng
Options

优化账单抓取

parent d60c2315
Showing with 253 additions and 104 deletions
AutoTurnOver.Models/dc_base_amazon_fulfilled_shipments.cs
......@@ -22,6 +22,7 @@ namespace AutoTurnOver.Models
public string buyer_name { get; set; }
public string buyer_phone_number { get; set; }
public string sku { get; set; }
public string product_name { get; set; }
public decimal? quantity_shipped { get; set; }
public string currency { get; set; }
......
AutoTurnOver.Services/AmazonDataSynchroService.cs
......@@ -315,7 +315,7 @@ namespace AutoTurnOver.Services
/// <summary>
/// 拉取报表
/// </summary>
public void SynchroReportIds(ReportTypes reportTypeEnum)
public void SynchroReportIds(ReportTypes reportTypeEnum,int version =8)
{
// 拉取亚马逊账号
var amazonAccounts = ApiServices.GetAmazonAccountList();
......@@ -328,14 +328,14 @@ namespace AutoTurnOver.Services
{
// 由于token1小时刷新一次，所以个账号要重新从账号系统拉取token
var item = ApiServices.GetAmazonAccountList().FirstOrDefault(s => s.Id == itemV.Id);
var task_name = $"SynchroAmazonReport_{reportTypeEnum.ToString()}_v8_{item.Id}";
var task_name = $"SynchroAmazonReport_{reportTypeEnum.ToString()}_v{version}_{item.Id}";
// 查询最后一次成功抓取的记录
var last_task_synchro_log = MyMySqlConnection._connection.QuerySingleOrDefault<task_synchro_log>(" select * from task_synchro_log where task_name=@task_name and status=1 order by end_time desc limit 1 ", new { task_name = task_name });
var new_task_synchro_log = new task_synchro_log
{
create_date = DateTime.Now,
end_time = DateTime.Now.AddMinutes(-3),
end_time = DateTime.Now.AddDays(-1),
status = 0,
task_name = task_name
};
......@@ -417,15 +417,111 @@ namespace AutoTurnOver.Services
}
}
public void SynchroReportIdByCreate(ReportTypes reportTypeEnum)
{
// 拉取亚马逊账号
var amazonAccounts = ApiServices.GetAmazonAccountList();
//amazonAccounts = amazonAccounts.Where(s => s.Account == "Natural Daisy Encens" && s.SiteEn == "UK").ToList();
foreach (var itemV in amazonAccounts)
{
try
{
if (itemV.AmznSpDevAccountObj != null && itemV.AmznSpTokenObj != null && itemV.MarketPlaceObj != null)
{
// 由于token1小时刷新一次，所以个账号要重新从账号系统拉取token
var item = ApiServices.GetAmazonAccountList().FirstOrDefault(s => s.Id == itemV.Id);
var task_name = $"SynchroAmazonReport_{reportTypeEnum.ToString()}_v9_{item.Id}";
// 查询最后一次成功抓取的记录
var last_task_synchro_log = MyMySqlConnection._connection.QuerySingleOrDefault<task_synchro_log>(" select * from task_synchro_log where task_name=@task_name and status=1 order by end_time desc limit 1 ", new { task_name = task_name });
var new_task_synchro_log = new task_synchro_log
{
create_date = DateTime.Now,
end_time = DateTime.Now.AddMinutes(-3),
status = 0,
task_name = task_name
};
if (last_task_synchro_log != null)
{
// 由于是拉报告，没必要太频繁，如果3小时之内拉过的，就先跳过
if ((DateTime.Now - last_task_synchro_log.end_time).TotalHours < 3)
{
continue;
}
new_task_synchro_log.start_time = last_task_synchro_log.end_time.AddMinutes(-1);
}
else
{
new_task_synchro_log.start_time = new DateTime(2023, 1, 1);
}
new_task_synchro_log.id = MyMySqlConnection._connection.Insert(new_task_synchro_log) ?? 0;
var reportId = CreateReport(item, new_task_synchro_log.start_time, new_task_synchro_log.end_time, reportTypeEnum);
var reportsDb = new dc_base_amazon_fee_report_log
{
ana_error = null,
ana_stack_trace = null,
ana_status = 0,
content = null,
create_time = DateTime.Now,
pams_account_id = item.Id,
report_id = reportId,
report_type = reportTypeEnum.ToString(),
//report_par_json = reportsItem.ToJson(),
btime = new_task_synchro_log.start_time,
etime = new_task_synchro_log.end_time,
site_code = item.MarketPlaceObj.multiMwsMid,
};
reportsDb.id = MyMySqlConnection._connection.QueryFirstOrDefault<int?>(" select id from dc_base_amazon_fee_report_log where report_id=@report_id and pams_account_id=@pams_account_id ", new
{
pams_account_id = reportsDb.pams_account_id,
report_id = reportsDb.report_id
}) ?? 0;
if (reportsDb.id <= 0)
{
MyMySqlConnection._connection.Insert(reportsDb);
}
new_task_synchro_log.count = 0;
new_task_synchro_log.status = 1;
MyMySqlConnection._connection.Update(new_task_synchro_log);
}
}
catch (Exception ex)
{
MyMySqlConnection._connection.Insert(new dc_task_error_log
{
date = DateTime.Now,
message = ex.Message,
stack_trace = ex.StackTrace,
task_name = "SynchroAmazonReport_" + itemV.Id
});
}
}
}
/// <summary>
/// 解析报告
/// </summary>
public void AnaReport()
public void AnaReport(int status = 0)
{
var reportTypeEnum = ReportTypes.GET_V2_SETTLEMENT_REPORT_DATA_XML;
var ana_task = MyMySqlConnection._connection.QueryFirstOrDefault<dc_base_amazon_fee_report_log>(" select * from dc_base_amazon_fee_report_log where report_type=@report_type and ana_status=0 order by id desc limit 1 ", new
var ana_task = MyMySqlConnection._connection.QueryFirstOrDefault<dc_base_amazon_fee_report_log>(" select * from dc_base_amazon_fee_report_log where report_type=@report_type and ana_status=@status order by id desc limit 1 ", new
{
report_type = reportTypeEnum.ToString()
report_type = reportTypeEnum.ToString(),
status = status
});
if (ana_task == null)
{
......@@ -566,58 +662,59 @@ namespace AutoTurnOver.Services
string jsonText = response_str.ToString();
ana_task.content = jsonText;
var table = CsvFileHelper.ReadFromCSV(filePath, true,'\t');
var table = CsvFileHelper.ReadFromCSV(filePath, true,jsonText.Contains("amazon-order-id") ?'\t':',');
foreach (DataRow row in table.Rows)
{
int comIndex = 0;
dc_base_amazon_fulfilled_shipments fulfilledShipments = new dc_base_amazon_fulfilled_shipments {
amazon_order_id = row.DataRowToString("amazon-order-id"),
amazon_order_item_id = row.DataRowToString("amazon-order-item-id"),
bill_address_1 = row.DataRowToString("bill-address-1"),
bill_address_2 = row.DataRowToString("bill-address-2"),
bill_address_3 = row.DataRowToString("bill-address-3"),
bill_city = row.DataRowToString("bill-city"),
bill_country = row.DataRowToString("bill-country"),
bill_postal_code = row.DataRowToString("bill-postal-code"),
bill_state = row.DataRowToString("bill-state"),
buyer_email = row.DataRowToString("buyer-email"),
buyer_name = row.DataRowToString("buyer-name"),
buyer_phone_number = row.DataRowToString("buyer-phone-number"),
carrier = row.DataRowToString("carrier"),
currency = row.DataRowToString("currency"),
estimated_arrival_date = row.DataRowToDateTime("estimated-arrival-date"),
fulfillment_center_id = row.DataRowToString("fulfillment-center-id"),
fulfillment_channel = row.DataRowToString("fulfillment-channel"),
gift_wrap_price = row.DataRowToNumber("gift-wrap-price"),
gift_wrap_tax = row.DataRowToNumber("gift-wrap-tax"),
item_price = row.DataRowToNumber("item-price"),
item_promotion_discount = row.DataRowToString("item-promotion-discount"),
item_tax = row.DataRowToNumber("item-tax"),
merchant_order_id = row.DataRowToString("merchant-order-id"),
payments_date = row.DataRowToDateTime("payments-date"),
merchant_order_item_id = row.DataRowToString("merchant-order-item-id"),
product_name = row.DataRowToString("product-name"),
purchase_date = row.DataRowToDateTime("purchase-date"),
quantity_shipped = row.DataRowToNumber("quantity-shipped"),
recipient_name = row.DataRowToString("recipient-name"),
reporting_date = row.DataRowToDateTime("reporting-date"),
sales_channel = row.DataRowToString("sales-channel"),
shipment_date = row.DataRowToDateTime("shipment-date"),
shipment_id = row.DataRowToString("shipment-id"),
shipment_item_id = row.DataRowToString("shipment-item-id"),
shipping_price = row.DataRowToNumber("shipping-price"),
shipping_tax = row.DataRowToNumber("shipping-tax"),
ship_address_1 = row.DataRowToString("ship-address-1"),
ship_address_2 = row.DataRowToString("ship-address-2"),
ship_address_3 = row.DataRowToString("ship-address-3"),
ship_city = row.DataRowToString("ship-city"),
ship_country = row.DataRowToString("ship-country"),
ship_phone_number = row.DataRowToString("ship-phone-number"),
ship_postal_code = row.DataRowToString("ship-postal-code"),
ship_promotion_discount = row.DataRowToString("ship-promotion-discount"),
ship_service_level = row.DataRowToString("ship-service-level"),
ship_state = row.DataRowToString("ship-state"),
sku = row.DataRowToString("sku"),
tracking_number = row.DataRowToString("tracking-number")
amazon_order_id = row.DataRowToString(comIndex++),
merchant_order_id = row.DataRowToString(comIndex++),
shipment_id = row.DataRowToString(comIndex++),
shipment_item_id = row.DataRowToString(comIndex++),
amazon_order_item_id = row.DataRowToString(comIndex++),
merchant_order_item_id = row.DataRowToString(comIndex++),
purchase_date = row.DataRowToDateTime(comIndex++),
payments_date = row.DataRowToDateTime(comIndex++),
shipment_date = row.DataRowToDateTime(comIndex++),
reporting_date = row.DataRowToDateTime(comIndex++),
buyer_email = row.DataRowToString(comIndex++),
buyer_name = row.DataRowToString(comIndex++),
buyer_phone_number = row.DataRowToString(comIndex++),
sku = row.DataRowToString(comIndex++),
product_name = row.DataRowToString(comIndex++),
quantity_shipped = row.DataRowToNumber(comIndex++),
currency = row.DataRowToString(comIndex++),
item_price = row.DataRowToNumber(comIndex++),
item_tax = row.DataRowToNumber(comIndex++),
shipping_price = row.DataRowToNumber(comIndex++),
shipping_tax = row.DataRowToNumber(comIndex++),
gift_wrap_price = row.DataRowToNumber(comIndex++),
gift_wrap_tax = row.DataRowToNumber(comIndex++),
ship_service_level = row.DataRowToString(comIndex++),
recipient_name = row.DataRowToString(comIndex++),
ship_address_1 = row.DataRowToString(comIndex++),
ship_address_2 = row.DataRowToString(comIndex++),
ship_address_3 = row.DataRowToString(comIndex++),
ship_city = row.DataRowToString(comIndex++),
ship_state = row.DataRowToString(comIndex++),
ship_postal_code = row.DataRowToString(comIndex++),
ship_country = row.DataRowToString(comIndex++),
ship_phone_number = row.DataRowToString(comIndex++),
bill_address_1 = row.DataRowToString(comIndex++),
bill_address_2 = row.DataRowToString(comIndex++),
bill_address_3 = row.DataRowToString(comIndex++),
bill_city = row.DataRowToString(comIndex++),
bill_state = row.DataRowToString(comIndex++),
bill_postal_code = row.DataRowToString(comIndex++),
bill_country = row.DataRowToString(comIndex++),
item_promotion_discount = row.DataRowToString(comIndex++),
ship_promotion_discount = row.DataRowToString(comIndex++),
carrier = row.DataRowToString(comIndex++),
tracking_number = row.DataRowToString(comIndex++),
estimated_arrival_date = row.DataRowToDateTime(comIndex++),
fulfillment_center_id = row.DataRowToString(comIndex++),
fulfillment_channel = row.DataRowToString(comIndex++),
sales_channel = row.DataRowToString(comIndex++)
};
fulfilledShipments.data_id = $"{fulfilledShipments.amazon_order_id}-{fulfilledShipments.amazon_order_item_id}-{fulfilledShipments.sku}-{fulfilledShipments.merchant_order_item_id}-{fulfilledShipments.shipment_id}-{fulfilledShipments.shipment_item_id}-end";
fulfilledShipments.id = MyMySqlConnection._connection.QueryFirstOrDefault<int?>(" select id from dc_base_amazon_fulfilled_shipments where data_id=@data_id ",new { data_id = fulfilledShipments.data_id })??0;
......@@ -720,7 +817,7 @@ namespace AutoTurnOver.Services
feeShipmentEvent.other = 0;
if (item.Promotion != null && item.Promotion.Amount.Text != 0)
if (item.Promotion != null && item.Promotion.Any(s=>s.Amount.Text != 0))
{
feeShipmentEvent.promotional_rebates = feeShipmentEvent.postage_credits;
feeShipmentEvent.promotional_rebates_tax = feeShipmentEvent.shipping_credits_tax;
......@@ -814,7 +911,7 @@ namespace AutoTurnOver.Services
feeShipmentEvent.other = 0;
if (itemFeeAdjustment.PromotionAdjustment != null && itemFeeAdjustment.PromotionAdjustment.Amount.Text != 0)
if (itemFeeAdjustment.PromotionAdjustment != null && itemFeeAdjustment.PromotionAdjustment.Any(s=>s.Amount.Text != 0))
{
feeShipmentEvent.promotional_rebates = feeShipmentEvent.postage_credits;
feeShipmentEvent.promotional_rebates_tax = feeShipmentEvent.shipping_credits_tax;
......
AutoTurnOver.Services/Dto/ApiAmzonResponseDto.cs
......@@ -150,6 +150,8 @@ namespace AutoTurnOver.Services.Dto
public DateTime? DepositDate { get; set; }
}
[JsonProperty("OtherTransaction")]
[JsonConverter(typeof(OtherTransactionDataConverter))]
public List<OtherTransactionDto> OtherTransaction { get; set; } = new List<OtherTransactionDto> { };
public class OtherTransactionDto
......@@ -209,13 +211,10 @@ namespace AutoTurnOver.Services.Dto
}
public PromotionDto Promotion { get; set; }
public class PromotionDto
{
public string MerchantPromotionID { get; set; }
public string Type { get; set; }
public Amount Amount { get; set; }
}
[JsonProperty("Promotion")]
[JsonConverter(typeof(PromotionDataConverter))]
public List<PromotionDto> Promotion { get; set; }
}
}
}
......@@ -260,13 +259,10 @@ namespace AutoTurnOver.Services.Dto
}
public PromotionDto PromotionAdjustment { get; set; }
public class PromotionDto
{
public string MerchantPromotionID { get; set; }
public string Type { get; set; }
public Amount Amount { get; set; }
}
[JsonProperty("PromotionAdjustment")]
[JsonConverter(typeof(PromotionDataConverter))]
public List<PromotionDto> PromotionAdjustment { get; set; }
}
}
}
......@@ -285,6 +281,12 @@ namespace AutoTurnOver.Services.Dto
}
}
}
public class PromotionDto
{
public string MerchantPromotionID { get; set; }
public string Type { get; set; }
public Amount Amount { get; set; }
}
public class FeeDto
{
public string Type { get; set; }
......@@ -330,6 +332,36 @@ namespace AutoTurnOver.Services.Dto
serializer.Serialize(writer, value);
}
}
class OtherTransactionDataConverter : JsonConverter
{
public override bool CanConvert(Type objectType)
{
return (objectType == typeof(ApiAmazonSettlementDto.AmazonEnvelopeDto.MessageDto.SettlementReportDto.OtherTransactionDto));
}
public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer)
{
JToken token = JToken.Load(reader);
if (token.Type == JTokenType.Array)
{
return token.ToObject<List<ApiAmazonSettlementDto.AmazonEnvelopeDto.MessageDto.SettlementReportDto.OtherTransactionDto>>();
}
if (token.Type == JTokenType.Object)
{
return new List<ApiAmazonSettlementDto.AmazonEnvelopeDto.MessageDto.SettlementReportDto.OtherTransactionDto> { token.ToObject<ApiAmazonSettlementDto.AmazonEnvelopeDto.MessageDto.SettlementReportDto.OtherTransactionDto>() };
}
else
{
return new List<ApiAmazonSettlementDto.AmazonEnvelopeDto.MessageDto.SettlementReportDto.OtherTransactionDto> { };
}
}
public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer)
{
serializer.Serialize(writer, value);
}
}
class RefundDataConverter : JsonConverter
{
public override bool CanConvert(Type objectType)
......@@ -453,6 +485,38 @@ namespace AutoTurnOver.Services.Dto
}
}
class PromotionDataConverter : JsonConverter
{
public override bool CanConvert(Type objectType)
{
return (objectType == typeof(ApiAmazonSettlementDto.AmazonEnvelopeDto.PromotionDto));
}
public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer)
{
JToken token = JToken.Load(reader);
if (token.Type == JTokenType.Array)
{
var test = token.ToString();
return token.ToObject<List<ApiAmazonSettlementDto.AmazonEnvelopeDto.PromotionDto>>();
}
if (token.Type == JTokenType.Object)
{
return new List<ApiAmazonSettlementDto.AmazonEnvelopeDto.PromotionDto> { token.ToObject<ApiAmazonSettlementDto.AmazonEnvelopeDto.PromotionDto> () };
}
else
{
return new List<ApiAmazonSettlementDto.AmazonEnvelopeDto.FeeDto> { };
}
}
public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer)
{
serializer.Serialize(writer, value);
}
}
class AdjustedItemDataConverter : JsonConverter
{
public override bool CanConvert(Type objectType)
......
AutoTurnOver.Utility/DataTableExtensions.cs
......@@ -42,54 +42,41 @@ namespace AutoTurnOver.Utility
}
public static string DataRowToString(this DataRow row, string key)
public static string DataRowToString(this DataRow row,int index)
{
if (row[key] == null)
if (row[index] == null)
{
return null;
}
else
{
return row[key].ToString();
return row[index].ToString();
}
}
public static DateTime? DataRowToDateTime(this DataRow row, string key)
public static DateTime? DataRowToDateTime(this DataRow row,int index)
{
try
if (row[index] == null)
{
if (row[key] == null)
{
return null;
}
else
{
return DateTime.Parse( row[key].ToString());
}
return null;
}
catch (Exception ex)
else
{
return null;
return DateTime.Parse(row[index].ToString());
}
}
public static decimal? DataRowToNumber(this DataRow row, string key)
public static decimal? DataRowToNumber(this DataRow row,int index)
{
try
if (row[index] == null)
{
if (row[key] == null)
{
return null;
}
else
{
return decimal.Parse( row[key].ToString());
}
return null;
}
catch (Exception ex)
else
{
return null;
return decimal.Parse(row[index].ToString());
}
}
......
ResetOutofstock/Program.cs
......@@ -54,10 +54,10 @@ namespace ResetOutofstock
//new AmazonDataSynchroService().SynchroFinancialEventGroups();
//new AmazonDataSynchroService().SynchroReportIds();
//new AmazonDataSynchroService().SynchroReportIds(FikaAmazonAPI.Utils.Constants.ReportTypes.GET_AMAZON_FULFILLED_SHIPMENTS_DATA_GENERAL);
//new AmazonDataSynchroService().SynchroReportIds(FikaAmazonAPI.Utils.Constants.ReportTypes.GET_AMAZON_FULFILLED_SHIPMENTS_DATA_GENERAL,10);
//while (true)
//{
// new AmazonDataSynchroService().AnaFbaShipmentReport();
// new AmazonDataSynchroService().AnaReport(-1);
//}
}
catch (Exception ex)
......
ResetOutofstock/ResetOutofstockBackgrounService.cs
......@@ -623,7 +623,7 @@ namespace ResetOutofstock
try
{
Console.WriteLine($"开始 拉取亚马逊费用流水，线程Id：{Thread.CurrentThread.ManagedThreadId}，{DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss")}");
new AmazonDataSynchroService().SynchroReportIds(FikaAmazonAPI.Utils.Constants.ReportTypes.GET_V2_SETTLEMENT_REPORT_DATA_XML);
new AmazonDataSynchroService().SynchroReportIds(FikaAmazonAPI.Utils.Constants.ReportTypes.GET_V2_SETTLEMENT_REPORT_DATA_XML,9);
Console.WriteLine($"结束 拉取亚马逊费用流水，线程Id：{Thread.CurrentThread.ManagedThreadId}，{DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss")}");
Thread.Sleep(1000 * 60 * 3 );
}
......@@ -645,7 +645,7 @@ namespace ResetOutofstock
try
{
Console.WriteLine($"开始 拉取亚马逊 fba发货，线程Id：{Thread.CurrentThread.ManagedThreadId}，{DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss")}");
new AmazonDataSynchroService().SynchroReportIds(FikaAmazonAPI.Utils.Constants.ReportTypes.GET_AMAZON_FULFILLED_SHIPMENTS_DATA_GENERAL);
new AmazonDataSynchroService().SynchroReportIds(FikaAmazonAPI.Utils.Constants.ReportTypes.GET_AMAZON_FULFILLED_SHIPMENTS_DATA_GENERAL,9);
Console.WriteLine($"结束 拉取亚马逊 fba发货，线程Id：{Thread.CurrentThread.ManagedThreadId}，{DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss")}");
Thread.Sleep(1000 * 60 * 3 );
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment