Skip to content

B
bailuntec-datacenter
Commit 12076186 by huluobin
Options

# update

parent 0079143a
Showing with 222 additions and 45 deletions
datacenter-job/datacenter-base/base-sync-ebay/src/main/java/com/bailuntec/job/EbayFinanceWriter.java
......@@ -2,10 +2,9 @@ package com.bailuntec.job;
import com.bailuntec.application.IDcBaseFinanceEbayService;
import com.bailuntec.domain.DcBaseFinanceEbay;
import lombok.extern.slf4j.Slf4j;
import org.springframework.batch.item.ItemWriter;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.lang.NonNull;
import org.springframework.stereotype.Component;
import java.util.List;
import java.util.stream.Collectors;
......@@ -18,6 +17,7 @@ import java.util.stream.Collectors;
* @author robbendev
* @since 2021/1/19 1:21 下午
*/
@Slf4j
public class EbayFinanceWriter implements ItemWriter<DcBaseFinanceEbay> {
final IDcBaseFinanceEbayService dcBaseFinanceEbayService;
......@@ -27,7 +27,8 @@ public class EbayFinanceWriter implements ItemWriter<DcBaseFinanceEbay> {
}
@Override
public void write(@NonNull List<? extends DcBaseFinanceEbay> items) throws Exception {
public void write(@NonNull List<? extends DcBaseFinanceEbay> items) {
log.info("批量写入DcBaseFinanceEbay");
dcBaseFinanceEbayService.saveBatch(items.stream().map(var -> (DcBaseFinanceEbay) var).collect(Collectors.toList()));
}
}
datacenter-job/datacenter-base/base-sync-ebay/src/main/java/com/bailuntec/job/EbayItemReader.java
package com.bailuntec.job;
import com.bailuntec.common.JsonUtilByFsJson;
import com.bailuntec.common.ListUtil;
import com.bailuntec.domain.DcBaseCompanyAccount;
import com.bailuntec.domain.DcJobConfig;
import com.bailuntec.infrastructure.mapper.DcBaseCompanyAccountMapper;
import com.bailuntec.infrastructure.mapper.DcJobConfigMapper;
import com.ebay.sdk.ApiContext;
import com.ebay.sdk.ApiCredential;
import com.ebay.sdk.TimeFilter;
......@@ -48,65 +48,66 @@ public class EbayItemReader implements ItemReader<EbayItem> {
private DcJobConfig dcJobConfig;
private final DcBaseCompanyAccountMapper dcBaseCompanyAccountMapper;
private final DcJobConfigMapper dcJobConfigMapper;
@SneakyThrows
public EbayItemReader(DcBaseCompanyAccountMapper dcBaseCompanyAccountMapper,
DcJobConfigMapper dcJobConfigMapper) {
DcJobConfig dcJobConfig) {
this.dcBaseCompanyAccountMapper = dcBaseCompanyAccountMapper;
this.dcJobConfigMapper = dcJobConfigMapper;
this.dcJobConfig = dcJobConfig;
this.init();
}
private void init() throws Exception {
private void init() {
dcBaseCompanyAccountList = dcBaseCompanyAccountMapper.queryPageEbay();
dcJobConfig = dcJobConfigMapper.selectByName("base-sync-finance-ebay-ad");
this.pageNum = dcJobConfig.getPageNum();
this.nextAccount();
}
@Override
public EbayItem read() throws Exception {
public EbayItem read() {
EbayItem ebayItem = null;
if (ListUtil.isNotEmpty(accountEntryTypeList)) {
AccountEntryType accountEntryType = accountEntryTypeList.remove(0);
log.info("读取一条广告费用记录");
return new EbayItem(accountEntryType, dcBaseCompanyAccount);
ebayItem = new EbayItem(accountEntryType, dcBaseCompanyAccount);
} else {
//翻页
nextPage();
if (ListUtil.isNotEmpty(accountEntryTypeList)) {
AccountEntryType accountEntryType = accountEntryTypeList.remove(0);
log.info("读取一条广告费用记录");
return new EbayItem(accountEntryType, dcBaseCompanyAccount);
} else {
nextAccount();
if (ListUtil.isNotEmpty(accountEntryTypeList)) {
AccountEntryType accountEntryType = accountEntryTypeList.remove(0);
log.info("读取一条广告费用记录");
return new EbayItem(accountEntryType, dcBaseCompanyAccount);
} else {
return null;
}
ebayItem = new EbayItem(accountEntryType, dcBaseCompanyAccount);
}
}
if (ebayItem != null) {
log.info("读取一条广告费用记录:{}", JsonUtilByFsJson.beanToJson(ebayItem));
}
return ebayItem;
}
private void nextPage() throws Exception {
private void nextPage() {
this.pageNum++;
refreshPage();
}
private void nextAccount() throws Exception {
//翻页还是空 换账号
if (ListUtil.isEmpty(accountEntryTypeList) && ListUtil.isNotEmpty(dcBaseCompanyAccountList)) {
this.nextAccount();
}
}
private void nextAccount() {
this.pageNum = 1;
if (ListUtil.isNotEmpty(dcBaseCompanyAccountList)) {
dcBaseCompanyAccount = dcBaseCompanyAccountList.remove(0);
log.info("切换账号 切换后当前账号:{},剩余待跑账号数量:{}", dcBaseCompanyAccount.getAccountName(), dcBaseCompanyAccountList.size());
this.refreshAccountCall();
this.refreshPage();
}
if (ListUtil.isEmpty(accountEntryTypeList) && ListUtil.isNotEmpty(dcBaseCompanyAccountList)) {
this.nextAccount();
}
}
void refreshAccountCall() {
......@@ -136,10 +137,21 @@ public class EbayItemReader implements ItemReader<EbayItem> {
getAccountCall.setInvoiceDate(endCal);
}
private void refreshPage() throws Exception {
private void refreshPage() {
PaginationType paginationType = new PaginationType();
paginationType.setPageNumber(pageNum);
getAccountCall.setPagination(new PaginationType());
accountEntryTypeList = Lists.newArrayList(getAccountCall.getAccount());
getAccountCall.setPagination(paginationType);
log.info("请求ebay广告服务开始,当前账号:{}，当前页码:{}", dcBaseCompanyAccount.getAccountName(), pageNum);
try {
AccountEntryType[] accountEntryTypes = getAccountCall.getAccount();
if (accountEntryTypes != null && accountEntryTypes.length > 0) {
accountEntryTypeList = Lists.newArrayList(accountEntryTypes);
}
} catch (Exception ex) {
log.info("请求ebay广告服务异常,当前账号:{}，当前页码:{}", dcBaseCompanyAccount.getAccountName(), pageNum);
//todo log ebay exception
} finally {
log.info("请求ebay广告服务结束,当前账号:{}，当前页码:{}", dcBaseCompanyAccount.getAccountName(), pageNum);
}
}
}
datacenter-job/datacenter-base/base-sync-ebay/src/main/java/com/bailuntec/job/JobConfiguration.java
package com.bailuntec.job;
import com.bailuntec.api.bailuntec.oa.OaApi;
import com.bailuntec.application.IDcBaseFinanceEbayService;
import com.bailuntec.domain.DcBaseFinanceEbay;
import com.bailuntec.domain.DcJobConfig;
import com.bailuntec.infrastructure.mapper.DcBaseCompanyAccountMapper;
import com.bailuntec.infrastructure.mapper.DcJobConfigMapper;
import org.springframework.batch.core.Job;
......@@ -13,8 +16,6 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import javax.batch.api.listener.JobListener;
/**
* <p>
*
......@@ -29,28 +30,42 @@ public class JobConfiguration {
public final JobBuilderFactory jobBuilderFactory;
public final StepBuilderFactory stepBuilderFactory;
private final EbayItemReader ebayItemReader;
private final EbayItemProcessor ebayItemProcessor;
private final EbayFinanceWriter ebayFinanceWriter;
private final DcBaseCompanyAccountMapper dcBaseCompanyAccountMapper;
private final DcJobConfigMapper dcJobConfigMapper;
private final IDcBaseFinanceEbayService dcBaseFinanceEbayService;
private final OaApi oaApi;
private DcJobConfig dcJobConfig;
@Autowired
public JobConfiguration(JobBuilderFactory jobBuilderFactory, StepBuilderFactory stepBuilderFactory, EbayItemReader ebayItemReader, EbayItemProcessor ebayItemProcessor, EbayFinanceWriter ebayFinanceWriter, JobListener jobListener, DcBaseCompanyAccountMapper dcBaseCompanyAccountMapper, DcJobConfigMapper dcJobConfigMapper) {
public JobConfiguration(JobBuilderFactory jobBuilderFactory,
StepBuilderFactory stepBuilderFactory,
DcBaseCompanyAccountMapper dcBaseCompanyAccountMapper,
DcJobConfigMapper dcJobConfigMapper,
IDcBaseFinanceEbayService dcBaseFinanceEbayService, OaApi oaApi) {
this.jobBuilderFactory = jobBuilderFactory;
this.stepBuilderFactory = stepBuilderFactory;
this.ebayItemReader = ebayItemReader;
this.ebayItemProcessor = ebayItemProcessor;
this.ebayFinanceWriter = ebayFinanceWriter;
this.dcBaseCompanyAccountMapper = dcBaseCompanyAccountMapper;
this.dcJobConfigMapper = dcJobConfigMapper;
this.dcBaseFinanceEbayService = dcBaseFinanceEbayService;
this.oaApi = oaApi;
dcJobConfig = dcJobConfigMapper.selectByName("base-sync-finance-ebay-ad");
}
@Bean
EbayItemReader reader() {
return new EbayItemReader(dcBaseCompanyAccountMapper, dcJobConfigMapper);
return new EbayItemReader(dcBaseCompanyAccountMapper, dcJobConfig);
}
@Bean
EbayItemProcessor processor() {
return new EbayItemProcessor(dcJobConfig, oaApi);
}
@Bean
EbayFinanceWriter writer() {
return new EbayFinanceWriter(dcBaseFinanceEbayService);
}
@Bean
......@@ -67,8 +82,8 @@ public class JobConfiguration {
return stepBuilderFactory.get("getData")
.<EbayItem, DcBaseFinanceEbay>chunk(100) // <输入,输出> 。chunk通俗的讲类似于SQL的commit; 这里表示处理(processor)100条后写入(writer)一次。
.reader(reader()) //指定ItemReader
.processor(ebayItemProcessor) //指定ItemProcessor
.writer(ebayFinanceWriter) //指定ItemWriter
.processor(processor()) //指定ItemProcessor
.writer(writer()) //指定ItemWriter
.build();
}
......
datacenter-job/datacenter-base/base-sync-ebay/src/main/resources/application-test.yml
......@@ -10,6 +10,8 @@ spring:
driver-class-name: com.mysql.jdbc.Driver
main:
allow-bean-definition-overriding: true
batch:
initialize-schema: always
#mybatis plus 配置
mybatis-plus:
......@@ -29,3 +31,4 @@ feign:
readTimeout: 30000
server:
port: 1224
datacenter-job/datacenter-base/base-sync-ebay/src/main/resources/logback-spring.xml 0 → 100644
<?xml version="1.0" encoding="UTF-8"?>
<configuration scan="true" scanPeriod="60 seconds" debug="true">
<property name="contextName" value="bailuntec"/>
<property name="LOG_HOME" value="log"/>
<!-- 彩色日志 -->
<!-- 彩色日志依赖的渲染类 -->
<conversionRule conversionWord="clr" converterClass="org.springframework.boot.logging.logback.ColorConverter"/>
<conversionRule conversionWord="wex"
converterClass="org.springframework.boot.logging.logback.WhitespaceThrowableProxyConverter"/>
<conversionRule conversionWord="wEx"
converterClass="org.springframework.boot.logging.logback.ExtendedWhitespaceThrowableProxyConverter"/>
<!-- 彩色日志格式 -->
<property name="CONSOLE_LOG_PATTERN"
value="${CONSOLE_LOG_PATTERN:-%clr(%d{yyyy-MM-dd HH:mm:ss.SSS}){faint} %clr(${LOG_LEVEL_PATTERN:-%5p}) %clr(${PID:- }){magenta} %clr(---){faint} %clr([%15.15t]){faint} %clr(%X{traceId}) %clr(%-40.40logger{39}){cyan} %clr(:){faint} %m%n${LOG_EXCEPTION_CONVERSION_WORD:-%wEx}}"/>
<contextName>${contextName}</contextName>
   
<!-- info日志 appender -->
<appender name="INFO" class="ch.qos.logback.core.rolling.RollingFileAppender">
<!--日志回滚策略-->
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<!-- 按天回滚 daily -->
<fileNamePattern>${LOG_HOME}/${contextName}-info.%d{yyyy-MM-dd}.log</fileNamePattern>
<!-- 日志最大的历史 30天 -->
<maxHistory>7</maxHistory>
</rollingPolicy>
<encoder>
<pattern>${CONSOLE_LOG_PATTERN}</pattern>
<charset class="java.nio.charset.Charset">UTF-8</charset>
</encoder>
<!-- 只打印错误日志 -->
<filter class="ch.qos.logback.classic.filter.LevelFilter">
<level>INFO</level>
<onMatch>ACCEPT</onMatch>
<onMismatch>DENY</onMismatch>
</filter>
</appender>
<!--warn日志appender-->
<appender name="WARN" class="ch.qos.logback.core.rolling.RollingFileAppender">
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<FileNamePattern>${LOG_HOME}/${contextName}-warn.%d{yyyy-MM-dd}.log</FileNamePattern>
<maxHistory>7</maxHistory>
</rollingPolicy>
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
<pattern>${CONSOLE_LOG_PATTERN}</pattern>
<charset class="java.nio.charset.Charset">UTF-8</charset>
</encoder>
<filter class="ch.qos.logback.classic.filter.LevelFilter">
<level>WARN</level>
<onMatch>ACCEPT</onMatch>
<onMismatch>DENY</onMismatch>
</filter>
</appender>
<!--error日志appender-->
<appender name="ERROR" class="ch.qos.logback.core.rolling.RollingFileAppender">
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<FileNamePattern>${LOG_HOME}/${contextName}-error.%d{yyyy-MM-dd}.log</FileNamePattern>
<maxHistory>7</maxHistory>
</rollingPolicy>
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
<pattern>${CONSOLE_LOG_PATTERN}</pattern>
<charset class="java.nio.charset.Charset">UTF-8</charset>
</encoder>
<filter class="ch.qos.logback.classic.filter.LevelFilter"><!-- 只打印错误日志 -->
<level>ERROR</level>
<onMatch>ACCEPT</onMatch>
<onMismatch>DENY</onMismatch>
</filter>
</appender>
<!--控制台日志输出-->
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<pattern>${CONSOLE_LOG_PATTERN}</pattern>
<charset class="java.nio.charset.Charset">UTF-8</charset>
</encoder>
</appender>
<!-- 开发环境日志级别为DEBUG -->
<springProfile name="dev">
<logger name="com.ebay.sdk" level="ERROR"/>
<logger name="com.bailuntec" level="INFO"/>
<logger name="org.springframework" level="INFO"/>
<!--日志打印的包的范围，及分类日志文件存储 -->
<logger name="com.bailuntec" additivity="false">
<level value="INFO"/>
<appender-ref ref="STDOUT"/>
</logger>
<root level="INFO" additivity="false">
<appender-ref ref="STDOUT"/>
</root>
</springProfile>
<!-- 测试环境日志级别为INFO -->
<springProfile name="test">
<logger name="com.ebay.sdk" level="ERROR"/>
<logger name="com.bailuntec" level="DEBUG"/>
<logger name="org.springframework" level="INFO"/>
<!--日志打印的包的范围，及分类日志文件存储 -->
<logger name="com.bailuntec" additivity="false">
<level value="INFO"/>
<appender-ref ref="STDOUT"/>
<appender-ref ref="ERROR"/>
<appender-ref ref="WARN"/>
<appender-ref ref="INFO"/>
</logger>
<root level="INFO" additivity="false">
<appender-ref ref="STDOUT"/>
</root>
</springProfile>
<!-- 正式环境日志级别为INFO -->
<springProfile name="prod">
<logger name="com.ebay.sdk" level="ERROR"/>
<logger name="com.bailuntec" level="DEBUG"/>
<logger name="org.springframework" level="INFO"/>
<!--日志打印的包的范围，及分类日志文件存储 -->
<logger name="com.bailuntec" additivity="false">
<level value="INFO"/>
<appender-ref ref="STDOUT"/>
<appender-ref ref="ERROR"/>
<appender-ref ref="WARN"/>
<appender-ref ref="INFO"/>
</logger>
<root level="INFO" additivity="false">
<appender-ref ref="STDOUT"/>
</root>
</springProfile>
</configuration>
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment