Skip to content

D
dc-java
Commit aff23183 by huluobin
Options

update

parent 9c4b48f8
Showing with 117 additions and 24 deletions
data-base/base-sync-finance-ebay/src/main/docker/Dockerfile
FROM openjdk:8-jdk-alpine
VOLUME /tmp
ADD base-sync-finance-ebay-1.0-SNAPSHOT.jar app.jar
ADD base-sync-finance-ebay-1.0-SNAPSHOT.jar base-sync-finance-ebay-1.0-SNAPSHOT.jar
RUN echo "Asia/Shanghai" > /etc/timezone
ENTRYPOINT ["java","-Xms600m","-Xmx3800m","-jar","/app.jar"]
ENTRYPOINT ["java","-Xms2G","-Xmx2G","-jar","/base-sync-finance-ebay-1.0-SNAPSHOT.jar"]
EXPOSE 8080
data-base/base-sync-finance-ebay/src/main/java/com/bailuntec/Application.java
......@@ -29,7 +29,9 @@ public class Application {
}
private static CoordinatorRegistryCenter createRegistryCenter() {
CoordinatorRegistryCenter regCenter = new ZookeeperRegistryCenter(new ZookeeperConfiguration(propertiesUtil.getPropertyAsString("ZOOKEEPER_SERVER"), propertiesUtil.getPropertyAsString("NAME_SPACE")));
CoordinatorRegistryCenter regCenter = new ZookeeperRegistryCenter(
new ZookeeperConfiguration(propertiesUtil.getPropertyAsString("ZOOKEEPER_SERVER"),
propertiesUtil.getPropertyAsString("NAME_SPACE")));
regCenter.init();
return regCenter;
}
......
data-base/base-sync-finance-ebay/src/main/java/com/bailuntec/job/EbayFinanceAdSyncJob.java
......@@ -26,10 +26,7 @@ import lombok.extern.slf4j.Slf4j;
import java.io.IOException;
import java.math.BigDecimal;
import java.time.Instant;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.ZoneId;
import java.time.*;
import java.util.Calendar;
import java.util.Date;
import java.util.List;
......@@ -48,12 +45,12 @@ public class EbayFinanceAdSyncJob extends PointJob {
List<JobAccountLog> listByPage = jobAccountLogMapper.getListByPageEbay(12);
LocalDateTime startTime = jobPointLog.getStartTime();
LocalDateTime endTime = jobPointLog.getEndTime();
LocalDateTime startTime = LocalDateTime.of(jobPointLog.getStartTime().toLocalDate(), LocalTime.of(0, 0));
LocalDateTime endTime = startTime.plusDays(1);
if (startTime.toLocalDate().compareTo(LocalDate.now()) < 0) {
listByPage.forEach(jobAccountLog -> {
listByPage.parallelStream().forEach(jobAccountLog -> {
//先删除再更新
dcBaseFinanceEbayMapper.deleteByExample(DcBaseFinanceEbayExample.newAndCreateCriteria()
.andBjDateBetween(startTime, endTime)
......
data-base/base-sync-finance-ebay/src/main/resources/job.properties
......@@ -10,6 +10,6 @@ ZOOKEEPER_SERVER=172.31.255.120:2181
NAME_SPACE=data-center
JOB_NAME=base-sync-finance-ebay-ad
JOB_CRON=0 0/1 * * * ?
JOB_CRON=0 8 * * * ?
SHARDING_TOTAL_COUNT=1
data-base/base-sync-finance-ebay/src/test/java/com/bailuntec/job/EbayFinanceAdSyncJobTest.java
......@@ -18,6 +18,7 @@ import org.junit.Test;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.time.temporal.ChronoUnit;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
......@@ -106,14 +107,17 @@ public class EbayFinanceAdSyncJobTest {
public void testThreadPool2() {
EbayFinanceAdSyncJob ebayFinanceAdSyncJob = new EbayFinanceAdSyncJob();
LocalDateTime start = LocalDateTime.of(2020, 11, 18, 0, 0);
LocalDateTime end = LocalDateTime.of(2020, 12, 2, 0, 0);
LocalDateTime start = LocalDateTime.of(2020, 12, 14, 0, 0);
LocalDateTime end = LocalDateTime.of(2020, 12, 14, 0, 0);
List<LocalDateTime> localDateTimeList = Stream.iterate(start,
localDateTime -> localDateTime.plusDays(1))
.limit(ChronoUnit.DAYS.between(start, end) + 1)
.collect(Collectors.toList());
Collections.reverse(localDateTimeList);
localDateTimeList.forEach(localDateTime -> {
JobPointLog jobPointLog = new JobPointLog();
jobPointLog.setStartTime(localDateTime);
......
data-base/base-sync-finance-ebay/src/test/java/com/bailuntec/job/EbayFinanceExport.java
......@@ -4,8 +4,6 @@ import com.alibaba.excel.annotation.ExcelProperty;
import lombok.Data;
import java.math.BigDecimal;
import java.time.LocalDateTime;
import java.util.Date;
@Data
public class EbayFinanceExport {
......@@ -13,7 +11,6 @@ public class EbayFinanceExport {
@ExcelProperty("费用类型")
private String description;
@ExcelProperty("日期")
private String bjDate;
......
data-common/src/main/java/com/bailuntec/mapper/DcBaseStockMapper.java
......@@ -233,7 +233,7 @@ public interface DcBaseStockMapper {
@Param("pageOffset") int pageOffset);
List<DcBaseStock> listNeNormal(ShardingContext shardingContext, int i, Integer pageSize);
List<DcBaseStock> listNeNormal(@Param("shardingContext") ShardingContext shardingContext, int i, Integer pageSize);
long countNeNormal(@Param("shardingContext") ShardingContext shardingContext);
}
data-common/src/main/resources/db-dev.properties
driver=com.mysql.jdbc.Driver
url=jdbc:mysql://gz-cdb-kp7s5i79.sql.tencentcdb.com:61691/bailun_datacenter?useUnicode=true&characterEncoding=utf-8&zeroDateTimeBehavior=convertToNull
url=jdbc:mysql://gz-cdb-kp7s5i79.sql.tencentcdb.com:61691/bailun_datacenter?useUnicode=true&characterEncoding=utf-8&zeroDateTimeBehavior=convertToNull&rewriteBatchedStatements=true
username=root
password=#7kfnymAM$Y9-Ntf
......
data-common/src/main/resources/mybatis-config.xml
......@@ -5,8 +5,8 @@
<configuration>
<!--<properties resource="db-tj.properties"/>-->
<!-- <properties resource="db-dev.properties"/>-->
<properties resource="db-prod.properties"/>
<properties resource="db-dev.properties"/>
<!-- <properties resource="db-prod.properties"/>-->
<settings>
<setting name="mapUnderscoreToCamelCase" value="true"/>
</settings>
......
data-show/show-auto-turnover/src/main/resources/job.properties
......@@ -10,14 +10,14 @@ EVENT_RDB_STORAGE_PASSWORD=#7kfnymAM$Y9-Ntf
ZOOKEEPER_SERVER=172.31.255.120:2181
NAME_SPACE=data-center
JOB_NAME=show-auto-turnover29
#JOB_CRON=0 0 3 * * ?
JOB_CRON=0 35 11 * * ?
JOB_NAME=show-auto-turnover30
JOB_CRON=0 0 3 * * ?
#JOB_CRON=0 16 8 * * ?
SHARDING_TOTAL_COUNT=8
JOB_NAME_NE_NORMAL=show-auto-turnover-ne-normal
JOB_CRON_NE_NORMAL=0 0 1 ? * MON
JOB_CRON_NE_NORMAL=0 0 10 ? * MON
JOB_NAME_INCREMENT=show-auto-turnover-increment3
......
data-show/show-auto-turnover/src/test/java/AutoTurnoverTest.java
import com.alibaba.excel.EasyExcel;
import com.alibaba.excel.annotation.ExcelProperty;
import com.alibaba.excel.context.AnalysisContext;
import com.alibaba.excel.event.AnalysisEventListener;
import com.alibaba.fastjson.JSON;
import com.bailuntec.domain.constant.CommonConstant;
import com.bailuntec.domain.entity.DcBaseStock;
import com.bailuntec.domain.entity.JobPointLog;
import com.bailuntec.domain.entity.SalesDayConfig;
import com.bailuntec.domain.example.DcAutoTurnoverExample;
import com.bailuntec.domain.example.DcBaseStockExample;
import com.bailuntec.domain.example.DcMidTransitExample;
import com.bailuntec.excel.DcAutoTurnoverExcel;
import com.bailuntec.excel.ExcelListener;
import com.bailuntec.job.AutoTurnoverJob;
import com.bailuntec.mapper.DcAutoTurnoverMapper;
import com.bailuntec.mapper.DcBaseStockMapper;
import com.bailuntec.mapper.DcMidTransitMapper;
import com.bailuntec.mapper.SalesDayConfigMapper;
import com.bailuntec.utils.PropertiesUtil;
import com.bailuntec.utils.SessionUtil;
......@@ -16,12 +24,20 @@ import com.dangdang.ddframe.job.executor.ShardingContexts;
import com.dangdang.ddframe.job.lite.lifecycle.api.JobAPIFactory;
import com.dangdang.ddframe.job.lite.lifecycle.api.JobOperateAPI;
import com.google.common.base.Optional;
import lombok.Data;
import org.apache.ibatis.session.ExecutorType;
import org.apache.ibatis.session.SqlSession;
import org.junit.jupiter.api.Test;
import org.junit.platform.commons.util.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.Serializable;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.Period;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
......@@ -133,11 +149,88 @@ public class AutoTurnoverTest {
System.out.println(interval);
}
@Test
public void test66() {
String filePath = "/Users/huluobin/Library/Containers/com.tencent.WeWorkMac/Data/Library/Application Support/WXWork/Data/1688853793439460/Cache/File/2020-12/广州-01仓停售停止监控部分AIMS需清除1128.xlsx";
EasyExcel.read(filePath, RemoveSku.class, new RemoveSkuListener()).sheet().doRead();
}
static class RemoveSkuListener extends AnalysisEventListener<RemoveSku> {
private static final Logger LOGGER = LoggerFactory.getLogger(RemoveSkuListener.class);
/**
* 每隔5条存储数据库，实际使用中可以3000条，然后清理list ，方便内存回收
*/
private static final int BATCH_COUNT = 1000;
List<RemoveSku> list = new ArrayList<>();
@Override
public void invoke(RemoveSku data, AnalysisContext context) {
LOGGER.info("解析到一条数据:{}", JSON.toJSONString(data));
list.add(data);
if (list.size() >= BATCH_COUNT) {
syncData();
list.clear();
}
}
@Override
public void doAfterAllAnalysed(AnalysisContext context) {
syncData();
LOGGER.info("所有数据解析完成！");
}
/**
* 加上存储数据库
*/
private void syncData() {
LOGGER.info("{}条数据，开始存储数据库！", list.size());
SqlSession sqlSession = SessionUtil.getFactory().openSession(ExecutorType.BATCH);
try {
DcBaseStockMapper dcBaseStockMapper = sqlSession.getMapper(DcBaseStockMapper.class);
DcMidTransitMapper dcMidTransitMapper = sqlSession.getMapper(DcMidTransitMapper.class);
DcAutoTurnoverMapper dcAutoTurnoverMapper = sqlSession.getMapper(DcAutoTurnoverMapper.class);
list.forEach(item -> {
if (StringUtils.isNotBlank(item.getBailunSku()) && StringUtils.isNotBlank(item.getWarehouseCode())) {
dcBaseStockMapper.deleteByExample(DcBaseStockExample.newAndCreateCriteria()
.andWarehouseCodeEqualTo(item.getWarehouseCode())
.andBailunSkuEqualTo(item.getBailunSku())
.example());
dcMidTransitMapper.deleteByExample(DcMidTransitExample.newAndCreateCriteria()
.andWarehouseCodeEqualTo(item.getWarehouseCode())
.andBailunSkuEqualTo(item.getBailunSku())
.example());
dcAutoTurnoverMapper.deleteByExample(DcAutoTurnoverExample.newAndCreateCriteria()
.andWarehouseCodeEqualTo(item.getWarehouseCode())
.andBailunSkuEqualTo(item.getBailunSku())
.example());
}
});
} finally {
sqlSession.commit();
sqlSession.close();
}
LOGGER.info("存储数据库成功！");
}
}
@Data
public static class RemoveSku implements Serializable {
@ExcelProperty("id")
private Integer id;
@ExcelProperty("Sku")
private String bailunSku;
@ExcelProperty("仓库编码")
private String warehouseCode;
}
@Test
public void testDcAutoTurnover() {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment