Skip to content

B
bailuntec-datacenter
Commit 2b632e5e by huluobin
Options

# update

parent b0f2ee20
Showing with 314 additions and 30 deletions
datacenter-task/ebay-report-consumer/src/main/java/com/bailuntec/task/EbayReportConsumerTaskApp.java
......@@ -22,6 +22,7 @@ import org.springframework.context.annotation.ComponentScan;
@ComponentScan("com.bailuntec")
@EnableFeignClients(basePackages = {"com.bailuntec.api", "com.bailuntec.*.api"})
public class EbayReportConsumerTaskApp {
public static void main(String[] args) {
SpringApplication.run(EbayReportConsumerTaskApp.class, args);
}
......
datacenter-task/ebay-report-consumer/src/main/java/com/bailuntec/task/job/JobConfiguration.java
package com.bailuntec.task.job;
import com.bailuntec.domain.DcBaseCompanyAccount;
import com.bailuntec.domain.DcBaseFinanceEbay;
import com.bailuntec.domain.EbayAccountReportTask;
import com.bailuntec.task.listener.TaskListener;
import com.bailuntec.task.processor.CustomerProcessor;
import com.bailuntec.task.reader.CustomerReader;
import com.bailuntec.task.reader.EbayItem;
import com.bailuntec.task.writer.CustomerWriter;
import org.springframework.batch.core.Job;
import org.springframework.batch.core.Step;
......@@ -57,7 +59,7 @@ public class JobConfiguration {
@Bean
public Step handleDataStep() {
return stepBuilderFactory.get("getData")
.<DcBaseCompanyAccount, EbayAccountReportTask>chunk(100) // <输入,输出> 。chunk通俗的讲类似于SQL的commit; 这里表示处理(processor)100条后写入(writer)一次。
.<EbayItem, DcBaseFinanceEbay>chunk(100) // <输入,输出> 。chunk通俗的讲类似于SQL的commit; 这里表示处理(processor)100条后写入(writer)一次。
.reader(customerReader) //指定ItemReader
.processor(customerProcessor) //指定ItemProcessor
.writer(customerWriter)//指定ItemWriter
......
datacenter-task/ebay-report-consumer/src/main/java/com/bailuntec/task/processor/CustomerProcessor.java
package com.bailuntec.task.processor;
import com.bailuntec.api.bailuntec.oa.OaApi;
import com.bailuntec.domain.DcBaseCompanyAccount;
import com.bailuntec.domain.DcJobConfig;
import com.bailuntec.domain.EbayAccountReportTask;
import com.bailuntec.domain.DcBaseFinanceEbay;
import com.bailuntec.infrastructure.util.enumerate.CurrencyType;
import com.bailuntec.task.config.JobProperty;
import com.bailuntec.task.reader.EbayItem;
import com.ebay.soap.eBLBaseComponents.AccountEntryType;
import org.springframework.batch.item.ItemProcessor;
import org.springframework.lang.NonNull;
import org.springframework.stereotype.Component;
import javax.annotation.Resource;
import java.math.BigDecimal;
import java.time.LocalDateTime;
import java.time.LocalTime;
import java.time.ZoneId;
/**
* <p>
......@@ -21,22 +25,51 @@ import java.time.LocalTime;
* @since 2021/1/21 5:01 下午
*/
@Component
public class CustomerProcessor implements ItemProcessor<DcBaseCompanyAccount, EbayAccountReportTask> {
public class CustomerProcessor implements ItemProcessor<EbayItem, DcBaseFinanceEbay> {
@Resource
JobProperty jobProperty;
private JobProperty jobProperty;
@Resource
private OaApi oaApi;
@Override
public EbayAccountReportTask process(@NonNull DcBaseCompanyAccount item) {
EbayAccountReportTask ebayAccountReportTask = new EbayAccountReportTask();
public DcBaseFinanceEbay process(@NonNull EbayItem ebayItem) throws Exception {
DcBaseFinanceEbay dcBaseFinanceEbay = new DcBaseFinanceEbay();
AccountEntryType accountEntryType = ebayItem.getAccountEntryType();
DcBaseCompanyAccount dcBaseCompanyAccount = ebayItem.getDcBaseCompanyAccount();
ZoneId zone = ZoneId.systemDefault();
LocalDateTime gmt = LocalDateTime.ofInstant(accountEntryType.getDate().getTime().toInstant(), zone);
DcJobConfig dcJobConfig = jobProperty.getDcJobConfig();
dcBaseFinanceEbay.setGmtDate(gmt);
dcBaseFinanceEbay.setBjDate(dcBaseFinanceEbay.getGmtDate().minusHours(15L));
dcBaseFinanceEbay.setAccountEntryType(accountEntryType.getAccountDetailsEntryType() == null ? "" : accountEntryType.getAccountDetailsEntryType().value());
dcBaseFinanceEbay.setVatPercent(accountEntryType.getVATPercent());
dcBaseFinanceEbay.setItemId(accountEntryType.getItemID());
dcBaseFinanceEbay.setTransactionId(accountEntryType.getTransactionID());
dcBaseFinanceEbay.setOrderLineId(accountEntryType.getOrderLineItemID());
dcBaseFinanceEbay.setReceivedTopRatedDiscount(accountEntryType.isReceivedTopRatedDiscount());
dcBaseFinanceEbay.setDescription(accountEntryType.getDescription());
dcBaseFinanceEbay.setMemo(accountEntryType.getMemo());
dcBaseFinanceEbay.setRefNumber(accountEntryType.getRefNumber());
dcBaseFinanceEbay.setTitle(accountEntryType.getTitle());
dcBaseFinanceEbay.setReportDate(ebayItem.getEbayAccountReportTask().getEndTime().toLocalDate());
if (accountEntryType.getGrossDetailAmount() != null) {
dcBaseFinanceEbay.setGrossAmount(BigDecimal.valueOf(accountEntryType.getGrossDetailAmount().getValue()));
dcBaseFinanceEbay.setCurrency(accountEntryType.getGrossDetailAmount().getCurrencyID().value());
ebayAccountReportTask.setAccountId(item.getAccountId());
ebayAccountReportTask.setStartTime(LocalDateTime.of(dcJobConfig.getStartTime().toLocalDate(), LocalTime.MIN));
ebayAccountReportTask.setEndTime(LocalDateTime.of(dcJobConfig.getStartTime().toLocalDate(), LocalTime.MIN).plusDays(1));
dcBaseFinanceEbay.setExchangeRate(oaApi.getExchangeRateByCurAndDate(dcBaseFinanceEbay.getCurrency(), CurrencyType.CNY.value(), dcBaseFinanceEbay.getBjDate().toLocalDate()).getRate());
dcBaseFinanceEbay.setExchangeRateUsd(oaApi.getExchangeRateByCurAndDate(dcBaseFinanceEbay.getCurrency(), CurrencyType.USD.value(), dcBaseFinanceEbay.getBjDate().toLocalDate()).getRate());
}
if (accountEntryType.getGrossDetailAmount() != null) {
dcBaseFinanceEbay.setNetAmount(BigDecimal.valueOf(accountEntryType.getGrossDetailAmount().getValue()));
}
dcBaseFinanceEbay.setAccountId(dcBaseCompanyAccount.getAccountId());
dcBaseFinanceEbay.setCompanyId(dcBaseCompanyAccount.getCompanyId());
dcBaseFinanceEbay.setBjModified(LocalDateTime.now());
return ebayAccountReportTask;
return dcBaseFinanceEbay;
}
}
datacenter-task/ebay-report-consumer/src/main/java/com/bailuntec/task/reader/CustomerReader.java
......@@ -11,8 +11,14 @@ import com.bailuntec.infrastructure.mapper.DcBaseCompanyAccountMapper;
import com.bailuntec.infrastructure.mapper.EbayAccountReportTaskMapper;
import com.bailuntec.task.config.JobProperty;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.ebay.sdk.ApiContext;
import com.ebay.sdk.ApiCredential;
import com.ebay.sdk.TimeFilter;
import com.ebay.sdk.call.GetAccountCall;
import com.ebay.soap.eBLBaseComponents.AccountEntryType;
import com.ebay.soap.eBLBaseComponents.AccountHistorySelectionCodeType;
import com.ebay.soap.eBLBaseComponents.DetailLevelCodeType;
import com.ebay.soap.eBLBaseComponents.PaginationType;
import com.google.common.collect.Lists;
import lombok.extern.slf4j.Slf4j;
import org.springframework.batch.item.ItemReader;
......@@ -40,6 +46,8 @@ public class CustomerReader implements ItemReader<EbayItem> {
private List<EbayAccountReportTask> ebayAccountReportTaskList;
//剩余广告记录
private List<AccountEntryType> accountEntryTypeList;
//当前任务
private EbayAccountReportTask ebayAccountReportTask;
//当前账户
private DcBaseCompanyAccount dcBaseCompanyAccount;
//当前ebay请求接口
......@@ -66,6 +74,7 @@ public class CustomerReader implements ItemReader<EbayItem> {
this.dcJobConfig = jobProperty.getDcJobConfig();
this.pageNum = dcJobConfig.getPageNum();
log.info("初始化reader 结束");
this.nextAccount();
......@@ -77,13 +86,13 @@ public class CustomerReader implements ItemReader<EbayItem> {
EbayItem ebayItem = null;
if (ListUtil.isNotEmpty(accountEntryTypeList)) {
AccountEntryType accountEntryType = accountEntryTypeList.remove(0);
ebayItem = new EbayItem(accountEntryType, dcBaseCompanyAccount);
ebayItem = new EbayItem(accountEntryType, dcBaseCompanyAccount, ebayAccountReportTask);
} else {
//翻页
nextPage();
if (ListUtil.isNotEmpty(accountEntryTypeList)) {
AccountEntryType accountEntryType = accountEntryTypeList.remove(0);
ebayItem = new EbayItem(accountEntryType, dcBaseCompanyAccount);
ebayItem = new EbayItem(accountEntryType, dcBaseCompanyAccount, ebayAccountReportTask);
}
}
if (ebayItem != null) {
......@@ -98,17 +107,20 @@ public class CustomerReader implements ItemReader<EbayItem> {
refreshPage();
//翻页还是空 换账号
if (ListUtil.isEmpty(accountEntryTypeList) && ListUtil.isNotEmpty(dcBaseCompanyAccountList)) {
if (ListUtil.isEmpty(accountEntryTypeList) && ListUtil.isNotEmpty(ebayAccountReportTaskList)) {
this.nextAccount();
}
}
private void nextAccount() {
this.pageNum = 1;
if (ListUtil.isNotEmpty(dcBaseCompanyAccountList)) {
dcBaseCompanyAccount = dcBaseCompanyAccountList.remove(0);
if (ListUtil.isNotEmpty(ebayAccountReportTaskList)) {
ebayAccountReportTask = ebayAccountReportTaskList.remove(0);
dcBaseCompanyAccount = dcBaseCompanyAccountMapper.selectOne(new LambdaQueryWrapper<DcBaseCompanyAccount>()
.eq(DcBaseCompanyAccount::getPlatformId, DcBaseCompanyAccount.EBAY_PLATFORM_ID)
.eq(DcBaseCompanyAccount::getAccountId, ebayAccountReportTask.getAccountId()));
log.info("切换账号 切换后当前账号:{},剩余待跑账号数量:{}", dcBaseCompanyAccount.getAccountName(), dcBaseCompanyAccountList.size());
log.info("切换任务 切换后当前账号:{},剩余待跑任务数量:{}", dcBaseCompanyAccount.getAccountName(), ebayAccountReportTaskList.size());
this.refreshAccountCall();
this.refreshPage();
......@@ -117,7 +129,7 @@ public class CustomerReader implements ItemReader<EbayItem> {
.between(DcBaseFinanceEbay::getBjDate, dcJobConfig.getStartTime(), dcJobConfig.getEndTime())
.eq(DcBaseFinanceEbay::getAccountId, dcBaseCompanyAccount.getAccountId()));
}
if (ListUtil.isEmpty(accountEntryTypeList) && ListUtil.isNotEmpty(dcBaseCompanyAccountList)) {
if (ListUtil.isEmpty(accountEntryTypeList) && ListUtil.isNotEmpty(ebayAccountReportTaskList)) {
this.nextAccount();
}
}
......@@ -136,11 +148,11 @@ public class CustomerReader implements ItemReader<EbayItem> {
//开始时间
Calendar startCal = Calendar.getInstance();
startCal.setTime(Date.from(dcJobConfig.getStartTime().atZone(ZoneId.systemDefault()).toInstant()));
startCal.setTime(Date.from(ebayAccountReportTask.getStartTime().atZone(ZoneId.systemDefault()).toInstant()));
//结束时间
Calendar endCal = Calendar.getInstance();
endCal.setTime(Date.from(dcJobConfig.getEndTime().atZone(ZoneId.systemDefault()).toInstant()));
endCal.setTime(Date.from(ebayAccountReportTask.getEndTime().atZone(ZoneId.systemDefault()).toInstant()));
TimeFilter timeFilter = new TimeFilter(startCal, endCal);
getAccountCall.setViewPeriod(timeFilter);
......
datacenter-task/ebay-report-consumer/src/main/java/com/bailuntec/task/reader/EbayItem.java
package com.bailuntec.task.reader;
import com.bailuntec.domain.DcBaseCompanyAccount;
import com.bailuntec.domain.EbayAccountReportTask;
import com.ebay.soap.eBLBaseComponents.AccountEntryType;
import lombok.AllArgsConstructor;
import lombok.Data;
......@@ -21,4 +22,5 @@ public class EbayItem {
private AccountEntryType accountEntryType;
private DcBaseCompanyAccount dcBaseCompanyAccount;
private EbayAccountReportTask ebayAccountReportTask;
}
datacenter-task/ebay-report-consumer/src/main/java/com/bailuntec/task/writer/CustomerWriter.java
package com.bailuntec.task.writer;
import com.bailuntec.application.IEbayAccountReportTaskService;
import com.bailuntec.domain.EbayAccountReportTask;
import com.bailuntec.application.IDcBaseFinanceEbayService;
import com.bailuntec.domain.DcBaseFinanceEbay;
import lombok.extern.slf4j.Slf4j;
import org.springframework.batch.item.ItemWriter;
import org.springframework.lang.NonNull;
import org.springframework.stereotype.Component;
import javax.annotation.Resource;
......@@ -20,14 +21,15 @@ import java.util.stream.Collectors;
*/
@Component
@Slf4j
public class CustomerWriter implements ItemWriter<EbayAccountReportTask> {
public class CustomerWriter implements ItemWriter<DcBaseFinanceEbay> {
@Resource
IEbayAccountReportTaskService ebayAccountReportTaskService;
private IDcBaseFinanceEbayService dcBaseFinanceEbayService;
@Override
public void write(List<? extends EbayAccountReportTask> items) {
log.info("批量写入EbayAccountReportTask");
ebayAccountReportTaskService.saveBatch(items.stream().map(var -> (EbayAccountReportTask) var).collect(Collectors.toList()));
@Override
public void write(@NonNull List<? extends DcBaseFinanceEbay> items) {
log.info("批量写入DcBaseFinanceEbay");
dcBaseFinanceEbayService.saveBatch(items.stream().map(var -> (DcBaseFinanceEbay) var).collect(Collectors.toList()));
}
}
datacenter-task/ebay-report-consumer/src/main/resources/application-prod.yml 0 → 100644
spring:
# 服务名称
application:
name: ebay-report-produce
# 数据配置
datasource-biz:
url: jdbc:mysql://10.0.8.15:3306/bailun_datacenter?serverTimezone=GMT%2B8&characterEncoding=utf-8&rewriteBatchedStatements=true
username: root
password: '#7kfnymAM$Y9-Ntf'
driver-class-name: com.mysql.jdbc.Driver
dataflow-datasource:
url: jdbc:mysql://${MYSQL_SERVICE_HOST}:${MYSQL_SERVICE_PORT}/mysql?passwordCharacterEncoding=utf-8
username: root
password: ${mysql-root-password}
driverClassName: org.mariadb.jdbc.Driver
testOnBorrow: true
validationQuery: "SELECT 1"
main:
allow-bean-definition-overriding: true
batch:
initialize-schema: always
cloud:
task:
closecontext-enabled: true
#mybatis plus 配置
mybatis-plus:
mapper-locations:
- classpath*:com/bailuntec/mapper/*.xml
# configuration:
# log-impl: org.apache.ibatis.logging.stdout.StdOutImpl
feign:
hystrix:
enabled: false
client:
config:
default:
connectTimeout: 30000
readTimeout: 30000
server:
port: 8080
datacenter-task/ebay-report-consumer/src/main/resources/application-test.yml 0 → 100644
spring:
# 服务名称
application:
name: ebay-report-produce
# 数据配置
datasource-biz:
url: jdbc:mysql://gz-cdb-kp7s5i79.sql.tencentcdb.com:61691/bailun_datacenter?serverTimezone=GMT%2B8&characterEncoding=utf-8&rewriteBatchedStatements=true
username: root
password: '#7kfnymAM$Y9-Ntf'
driver-class-name: com.mysql.jdbc.Driver
dataflow-datasource:
url: jdbc:mysql://gz-cdb-kp7s5i79.sql.tencentcdb.com:61691/bailun_datacenter?serverTimezone=GMT%2B8&characterEncoding=utf-8&rewriteBatchedStatements=true
username: root
password: '#7kfnymAM$Y9-Ntf'
driver-class-name: com.mysql.jdbc.Driver
main:
allow-bean-definition-overriding: true
batch:
initialize-schema: always
cloud:
task:
closecontext-enabled: true
#mybatis plus 配置
mybatis-plus:
mapper-locations:
- classpath*:com/bailuntec/mapper/*.xml
# configuration:
# log-impl: org.apache.ibatis.logging.stdout.StdOutImpl
feign:
hystrix:
enabled: false
client:
config:
default:
connectTimeout: 30000
readTimeout: 30000
server:
port: 1224
datacenter-task/ebay-report-consumer/src/main/resources/logback-spring.xml 0 → 100644
<?xml version="1.0" encoding="UTF-8"?>
<configuration scan="true" scanPeriod="60 seconds" debug="true">
<property name="contextName" value="bailuntec"/>
<property name="LOG_HOME" value="log"/>
<!-- 彩色日志 -->
<!-- 彩色日志依赖的渲染类 -->
<conversionRule conversionWord="clr" converterClass="org.springframework.boot.logging.logback.ColorConverter"/>
<conversionRule conversionWord="wex"
converterClass="org.springframework.boot.logging.logback.WhitespaceThrowableProxyConverter"/>
<conversionRule conversionWord="wEx"
converterClass="org.springframework.boot.logging.logback.ExtendedWhitespaceThrowableProxyConverter"/>
<!-- 彩色日志格式 -->
<property name="CONSOLE_LOG_PATTERN"
value="${CONSOLE_LOG_PATTERN:-%clr(%d{yyyy-MM-dd HH:mm:ss.SSS}){faint} %clr(${LOG_LEVEL_PATTERN:-%5p}) %clr(${PID:- }){magenta} %clr(---){faint} %clr([%15.15t]){faint} %clr(%X{traceId}) %clr(%-40.40logger{39}){cyan} %clr(:){faint} %m%n${LOG_EXCEPTION_CONVERSION_WORD:-%wEx}}"/>
<contextName>${contextName}</contextName>
   
<!-- info日志 appender -->
<appender name="INFO" class="ch.qos.logback.core.rolling.RollingFileAppender">
<!--日志回滚策略-->
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<!-- 按天回滚 daily -->
<fileNamePattern>${LOG_HOME}/${contextName}-info.%d{yyyy-MM-dd}.log</fileNamePattern>
<!-- 日志最大的历史 30天 -->
<maxHistory>7</maxHistory>
</rollingPolicy>
<encoder>
<pattern>${CONSOLE_LOG_PATTERN}</pattern>
<charset class="java.nio.charset.Charset">UTF-8</charset>
</encoder>
<!-- 只打印错误日志 -->
<filter class="ch.qos.logback.classic.filter.LevelFilter">
<level>INFO</level>
<onMatch>ACCEPT</onMatch>
<onMismatch>DENY</onMismatch>
</filter>
</appender>
<!--warn日志appender-->
<appender name="WARN" class="ch.qos.logback.core.rolling.RollingFileAppender">
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<FileNamePattern>${LOG_HOME}/${contextName}-warn.%d{yyyy-MM-dd}.log</FileNamePattern>
<maxHistory>7</maxHistory>
</rollingPolicy>
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
<pattern>${CONSOLE_LOG_PATTERN}</pattern>
<charset class="java.nio.charset.Charset">UTF-8</charset>
</encoder>
<filter class="ch.qos.logback.classic.filter.LevelFilter">
<level>WARN</level>
<onMatch>ACCEPT</onMatch>
<onMismatch>DENY</onMismatch>
</filter>
</appender>
<!--error日志appender-->
<appender name="ERROR" class="ch.qos.logback.core.rolling.RollingFileAppender">
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<FileNamePattern>${LOG_HOME}/${contextName}-error.%d{yyyy-MM-dd}.log</FileNamePattern>
<maxHistory>7</maxHistory>
</rollingPolicy>
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
<pattern>${CONSOLE_LOG_PATTERN}</pattern>
<charset class="java.nio.charset.Charset">UTF-8</charset>
</encoder>
<filter class="ch.qos.logback.classic.filter.LevelFilter"><!-- 只打印错误日志 -->
<level>ERROR</level>
<onMatch>ACCEPT</onMatch>
<onMismatch>DENY</onMismatch>
</filter>
</appender>
<!--控制台日志输出-->
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<pattern>${CONSOLE_LOG_PATTERN}</pattern>
<charset class="java.nio.charset.Charset">UTF-8</charset>
</encoder>
</appender>
<!-- 开发环境日志级别为DEBUG -->
<springProfile name="dev">
<logger name="com.ebay.sdk" level="ERROR"/>
<logger name="com.bailuntec" level="INFO"/>
<logger name="org.springframework" level="INFO"/>
<!--日志打印的包的范围，及分类日志文件存储 -->
<logger name="com.bailuntec" additivity="false">
<level value="INFO"/>
<appender-ref ref="STDOUT"/>
</logger>
<root level="INFO" additivity="false">
<appender-ref ref="STDOUT"/>
</root>
</springProfile>
<!-- 测试环境日志级别为INFO -->
<springProfile name="test">
<logger name="com.ebay.sdk" level="ERROR"/>
<logger name="com.bailuntec" level="DEBUG"/>
<logger name="org.springframework" level="INFO"/>
<!--日志打印的包的范围，及分类日志文件存储 -->
<logger name="com.bailuntec" additivity="false">
<level value="INFO"/>
<appender-ref ref="STDOUT"/>
<appender-ref ref="ERROR"/>
<appender-ref ref="WARN"/>
<appender-ref ref="INFO"/>
</logger>
<root level="INFO" additivity="false">
<appender-ref ref="STDOUT"/>
</root>
</springProfile>
<!-- 正式环境日志级别为INFO -->
<springProfile name="prod">
<logger name="com.ebay.sdk" level="ERROR"/>
<logger name="com.bailuntec" level="DEBUG"/>
<logger name="org.springframework" level="INFO"/>
<!--日志打印的包的范围，及分类日志文件存储 -->
<logger name="com.bailuntec" additivity="false">
<level value="INFO"/>
<appender-ref ref="STDOUT"/>
<appender-ref ref="ERROR"/>
<appender-ref ref="WARN"/>
<appender-ref ref="INFO"/>
</logger>
<root level="INFO" additivity="false">
<appender-ref ref="STDOUT"/>
</root>
</springProfile>
</configuration>
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment