Skip to content

D
dc-java
Commit d9aa158a by wutong
Options

Merge remote-tracking branch 'origin/master'

parents 2cc83aaf 68d12edb
Showing with 325 additions and 60 deletions
data-base/base_sync_finance_amazon/pom.xml
......@@ -37,6 +37,16 @@
<version>RELEASE</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>redis.clients</groupId>
<artifactId>jedis</artifactId>
<version>2.9.0</version>
<type>jar</type>
<scope>compile</scope>
</dependency>
</dependencies>
<build>
......
data-base/base_sync_finance_amazon/src/main/java/com/bailuntec/Application.java
package com.bailuntec;
import com.alibaba.druid.pool.DruidDataSource;
import com.bailuntec.job.AmazonFinanceSyncJob;
import com.bailuntec.job.AdvertAddTaskSyncJob;
import com.bailuntec.job.AdvertResultSyncJob;
import com.bailuntec.utils.PropertiesUtil;
import com.dangdang.ddframe.job.config.JobCoreConfiguration;
import com.dangdang.ddframe.job.config.simple.SimpleJobConfiguration;
......@@ -23,8 +24,10 @@ public class Application {
private static final String EVENT_RDB_STORAGE_URL = propertiesUtil.getPropertyAsString("EVENT_RDB_STORAGE_URL");
private static final String EVENT_RDB_STORAGE_USERNAME = propertiesUtil.getPropertyAsString("EVENT_RDB_STORAGE_USERNAME");
private static final String EVENT_RDB_STORAGE_PASSWORD = propertiesUtil.getPropertyAsString("EVENT_RDB_STORAGE_PASSWORD");
public static void main(String[] args) {
new JobScheduler(createRegistryCenter(), createJobConfiguration1()).init();
new JobScheduler(createRegistryCenter(), createJobConfiguration2()).init();
}
private static CoordinatorRegistryCenter createRegistryCenter() {
......@@ -34,8 +37,15 @@ public class Application {
}
private static LiteJobConfiguration createJobConfiguration1() {
JobCoreConfiguration simpleCoreConfig = JobCoreConfiguration.newBuilder(propertiesUtil.getPropertyAsString("JOB_NAME"), propertiesUtil.getPropertyAsString("JOB_CRON"), propertiesUtil.getPropertyAsInt("SHARDING_TOTAL_COUNT")).build();
SimpleJobConfiguration simpleJobConfig = new SimpleJobConfiguration(simpleCoreConfig, AmazonFinanceSyncJob.class.getCanonicalName());
JobCoreConfiguration simpleCoreConfig = JobCoreConfiguration.newBuilder(propertiesUtil.getPropertyAsString("JOB_NAME_ADDTASK"), propertiesUtil.getPropertyAsString("JOB_CRON_ADDTASK"), propertiesUtil.getPropertyAsInt("SHARDING_TOTAL_COUNT")).build();
SimpleJobConfiguration simpleJobConfig = new SimpleJobConfiguration(simpleCoreConfig, AdvertAddTaskSyncJob.class.getCanonicalName());
LiteJobConfiguration simpleJobRootConfig = LiteJobConfiguration.newBuilder(simpleJobConfig).build();
return simpleJobRootConfig;
}
private static LiteJobConfiguration createJobConfiguration2() {
JobCoreConfiguration simpleCoreConfig = JobCoreConfiguration.newBuilder(propertiesUtil.getPropertyAsString("JOB_NAME_GETRESULT"), propertiesUtil.getPropertyAsString("JOB_CRON_GETRESULT"), propertiesUtil.getPropertyAsInt("SHARDING_TOTAL_COUNT")).build();
SimpleJobConfiguration simpleJobConfig = new SimpleJobConfiguration(simpleCoreConfig, AdvertResultSyncJob.class.getCanonicalName());
LiteJobConfiguration simpleJobRootConfig = LiteJobConfiguration.newBuilder(simpleJobConfig).build();
return simpleJobRootConfig;
}
......
data-base/base_sync_finance_amazon/src/main/java/com/bailuntec/job/AmazonFinanceSyncJob.java → data-base/base_sync_finance_amazon/src/main/java/com/bailuntec/job/AdvertAddTaskSyncJob.java
package com.bailuntec.job;
import com.alibaba.fastjson.JSON;
import com.bailuntec.domain.AdvertResultData;
import com.alibaba.fastjson.JSONObject;
import com.bailuntec.domain.AdvertResultInfo;
import com.bailuntec.domain.AmazonAdvert;
import com.bailuntec.domain.entity.JobAccountLog;
import com.bailuntec.domain.entity.JobPointLog;
import com.bailuntec.domain.enumerate.AccountPlatformType;
import com.bailuntec.domain.example.DcBaseCompanyAccountExample;
import com.bailuntec.domain.example.JobAccountLogExample;
import com.bailuntec.mapper.DcAmazonAdvertMapper;
import com.bailuntec.mapper.DcBaseCompanyAccountMapper;
import com.bailuntec.mapper.JobAccountLogMapper;
import com.bailuntec.support.PointJob;
import com.bailuntec.utils.OkHttpUtil;
import com.bailuntec.utils.PropertiesUtil;
import com.bailuntec.utils.RedisUtil;
import com.bailuntec.utils.SessionUtil;
import com.dangdang.ddframe.job.api.ShardingContext;
import lombok.extern.slf4j.Slf4j;
......@@ -28,10 +27,11 @@ import java.util.HashMap;
import java.util.List;
@Slf4j
public class AmazonFinanceSyncJob extends PointJob {
public class AdvertAddTaskSyncJob extends PointJob {
private PropertiesUtil propertiesUtil = PropertiesUtil.getInstance("const");
private OkHttpClient client = OkHttpUtil.getInstance();
private RedisUtil redisUtil = new RedisUtil();
@Override
public void executeJob(ShardingContext shardingContext, JobPointLog jobPointLog) {
......@@ -58,7 +58,7 @@ public class AmazonFinanceSyncJob extends PointJob {
}
} catch (Exception e) {
e.printStackTrace();
log.warn("Amazon广告费定时同步费用接口错误", e);
log.warn("Amazon任务发布接口调用错误", e);
} finally {
SessionUtil.closeSession();
}
......@@ -120,9 +120,8 @@ public class AmazonFinanceSyncJob extends PointJob {
localDateTime = localDateTime.plusDays(1L);
} while (localDateTime.compareTo(job.getEndTime()) <= 0);
for (List<Integer> taskIdsList : taskIdsLists) {
callSaasApi(job, taskIdsList);
}
String taskIdsStr = JSONObject.toJSONString(taskIdsLists);
redisUtil.saveValueByKey(String.valueOf(job.getAccountId()), taskIdsStr);
job.setStartTime(job.getEndTime());
job.setEndTime(job.getStartTime().plusDays(job.getIntervalTime()).isAfter(LocalDateTime.now()) ? LocalDateTime.now() : job.getEndTime().plusDays(job.getIntervalTime()));
......@@ -136,49 +135,5 @@ public class AmazonFinanceSyncJob extends PointJob {
}
}
;
public void callSaasApi(JobAccountLog job, List<Integer> ids) {
HashMap<String, Object> map = new HashMap<>(2);
map.put("app_id", 1);
map.put("task_ids", ids);
MediaType mediaType = MediaType.parse("application/json");
Response response = null;
try {
DcAmazonAdvertMapper dcAmazonAdvertMapper = SessionUtil.getSession().getMapper(DcAmazonAdvertMapper.class);
RequestBody body = RequestBody.create(mediaType, JSON.toJSONString(map));
Request request = new Request.Builder()
.url(propertiesUtil.getPropertyAsString("GETRESULT_FINANCE_URL"))
.post(body)
.addHeader("Content-Type", "application/json")
.build();
response = client.newCall(request).execute();
if (response.isSuccessful()) {
AdvertResultInfo advertResult = JSON.parseObject(response.body().string(), AdvertResultInfo.class);
if (advertResult != null && advertResult.getCode() == 0) {
List<AdvertResultData> accountEntries = advertResult.getResults();
if (accountEntries != null && accountEntries.size() > 0) {
List<AmazonAdvert> advertList = JSON.parseArray(accountEntries.get(0).getData().get(0), AmazonAdvert.class);
for (AmazonAdvert amazonAdvert : advertList) {
amazonAdvert.setAccountId(job.getAccountId());
amazonAdvert.setCompanyId(job.getCompanyId());
dcAmazonAdvertMapper.upsertSelective(amazonAdvert);
}
}
} else {
throw new RuntimeException("授权错误");
}
} else {
throw new RuntimeException("接口调用失败");
}
} catch (Exception e) {
job.setMessage(e.getMessage());
} finally {
if (response != null) {
response.close();
}
}
}
}
data-base/base_sync_finance_amazon/src/main/java/com/bailuntec/job/AdvertResultSyncJob.java 0 → 100644
package com.bailuntec.job;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.bailuntec.domain.AdvertResultData;
import com.bailuntec.domain.AdvertResultInfo;
import com.bailuntec.domain.AmazonAdvert;
import com.bailuntec.domain.entity.JobAccountLog;
import com.bailuntec.domain.entity.JobPointLog;
import com.bailuntec.domain.enumerate.AccountPlatformType;
import com.bailuntec.domain.example.DcBaseCompanyAccountExample;
import com.bailuntec.mapper.DcAmazonAdvertMapper;
import com.bailuntec.mapper.DcBaseCompanyAccountMapper;
import com.bailuntec.mapper.JobAccountLogMapper;
import com.bailuntec.support.PointJob;
import com.bailuntec.utils.OkHttpUtil;
import com.bailuntec.utils.PropertiesUtil;
import com.bailuntec.utils.RedisUtil;
import com.bailuntec.utils.SessionUtil;
import com.dangdang.ddframe.job.api.ShardingContext;
import lombok.extern.slf4j.Slf4j;
import okhttp3.*;
import java.util.HashMap;
import java.util.List;
@Slf4j
public class AdvertResultSyncJob extends PointJob {
private PropertiesUtil propertiesUtil = PropertiesUtil.getInstance("const");
private OkHttpClient client = OkHttpUtil.getInstance();
private RedisUtil redisUtil = new RedisUtil();
@Override
public void executeJob(ShardingContext shardingContext, JobPointLog jobPointLog) {
int totalPage = getCount(jobPointLog);//分片即将处理的账号总数
try {
JobAccountLogMapper jobAccountLogMapper = SessionUtil.getSession().getMapper(JobAccountLogMapper.class);
int pageSize = totalPage % shardingContext.getShardingTotalCount() == 0 ? totalPage / shardingContext.getShardingTotalCount() : totalPage / shardingContext.getShardingTotalCount() + 1;
List<JobAccountLog> listByPage = jobAccountLogMapper.getListByPage(AccountPlatformType.Amazon.code(), pageSize * shardingContext.getShardingItem(), pageSize);
for (JobAccountLog jobAccountLog : listByPage) {
String taskIds = redisUtil.getValueByKey(String.valueOf(jobAccountLog.getAccountId()));
List<List<Integer>> taskIdsList = (List<List<Integer>>) JSONArray.parse(taskIds);
for (List<Integer> ids : taskIdsList) {
callSaasApi(jobAccountLog, ids);
}
}
} catch (Exception e) {
e.printStackTrace();
log.warn("Amazon广告费接口调用错误", e);
} finally {
SessionUtil.closeSession();
}
}
private int getCount(JobPointLog jobPointLog) {
int i = 0;
try {
DcBaseCompanyAccountMapper accountMapper = SessionUtil.getSession().getMapper(DcBaseCompanyAccountMapper.class);
i = (int) accountMapper.countByExample(DcBaseCompanyAccountExample.newAndCreateCriteria().andPlatformIdEqualTo(AccountPlatformType.Amazon.code()).example());
} catch (Exception e) {
e.printStackTrace();
} finally {
SessionUtil.closeSession();
}
return i;
}
public void callSaasApi(JobAccountLog job, List<Integer> ids) {
HashMap<String, Object> map = new HashMap<>(2);
map.put("app_id", 1);
map.put("task_ids", ids);
MediaType mediaType = MediaType.parse("application/json");
Response response = null;
try {
DcAmazonAdvertMapper dcAmazonAdvertMapper = SessionUtil.getSession().getMapper(DcAmazonAdvertMapper.class);
RequestBody body = RequestBody.create(mediaType, JSON.toJSONString(map));
Request request = new Request.Builder()
.url(propertiesUtil.getPropertyAsString("GETRESULT_FINANCE_URL"))
.post(body)
.addHeader("Content-Type", "application/json")
.build();
response = client.newCall(request).execute();
if (response.isSuccessful()) {
AdvertResultInfo advertResult = JSON.parseObject(response.body().string(), AdvertResultInfo.class);
if (advertResult != null && advertResult.getCode() == 0) {
List<AdvertResultData> accountEntries = advertResult.getResults();
if (accountEntries != null && accountEntries.size() > 0) {
List<AmazonAdvert> advertList = JSON.parseArray(accountEntries.get(0).getData().get(0), AmazonAdvert.class);
for (AmazonAdvert amazonAdvert : advertList) {
amazonAdvert.setAccountId(job.getAccountId());
amazonAdvert.setCompanyId(job.getCompanyId());
dcAmazonAdvertMapper.upsertSelective(amazonAdvert);
}
}
} else {
throw new RuntimeException("授权错误");
}
} else {
throw new RuntimeException("接口调用失败");
}
} catch (Exception e) {
job.setMessage(e.getMessage());
} finally {
if (response != null) {
response.close();
}
}
}
}
data-base/base_sync_finance_amazon/src/main/java/com/bailuntec/utils/RedisUtil.java 0 → 100644
package com.bailuntec.utils;
import lombok.extern.slf4j.Slf4j;
import redis.clients.jedis.Jedis;
import redis.clients.jedis.JedisPool;
import redis.clients.jedis.JedisPoolConfig;
@Slf4j
public class RedisUtil {
private static PropertiesUtil propertiesUtil = PropertiesUtil.getInstance("redis");
private static JedisPool jedisPool = null;
/*
* 初始化连接池
*/
static{
JedisPoolConfig config = new JedisPoolConfig();
config.setMaxTotal(propertiesUtil.getPropertyAsInt("SETMAXTOTAL"));
config.setMaxIdle(propertiesUtil.getPropertyAsInt("SETMAXIDLE"));
config.setMaxWaitMillis(propertiesUtil.getPropertyAsInt("SETMAXWAITMILLLIS"));
config.setTestOnBorrow(true);
jedisPool = new JedisPool(config, propertiesUtil.getPropertyAsString("REDIS_URL"), propertiesUtil.getPropertyAsInt("REDIS_PORT"));
log.warn("初始化连接池");
}
/*
* 获取redis实例
*/
public synchronized static Jedis getJedis() {
Jedis jedis = null;
try{
jedis =jedisPool.getResource();
}catch(Exception e) {
e.printStackTrace();
}
return jedis;
}
/*
* 回收redis对象资源
*/
public synchronized static void returnResource(Jedis jedis) {
if(jedis != null) {
try{
jedis.close();
}catch (Exception e){
e.printStackTrace();
}
}
}
/*
* 获取key值
*/
public String getValueByKey(String key) {
Jedis jedis = null;
String result = null;
try {
jedis = getJedis();
result = jedis.get(key);
} catch (Exception e) {
e.printStackTrace();
} finally {
returnResource(jedis);
}
return result;
}
/*
* 删除key
*/
public void deleteByKey(String key) throws Exception {
Jedis jedis = null;
try {
jedis = getJedis();
jedis.del(key);
} catch (Exception e) {
throw e;
} finally {
returnResource(jedis);
}
}
/*
* 设置过期时间值
*/
public void saveValueByKey(String key, String value) throws Exception {
Jedis jedis = null;
try {
jedis = getJedis();
jedis.set(key, value);
jedis.expire(key, propertiesUtil.getPropertyAsInt("EXPIRETIME"));
} catch (Exception e) {
throw e;
} finally {
returnResource(jedis);
}
}
}
data-base/base_sync_finance_amazon/src/main/resources/job.properties
......@@ -9,6 +9,8 @@ EVENT_RDB_STORAGE_USERNAME=root
EVENT_RDB_STORAGE_PASSWORD=#7kfnymAM$Y9-Ntf
ZOOKEEPER_SERVER=172.31.255.120:2181
NAME_SPACE=data-center
JOB_NAME=base-sync-finance-amazon
JOB_CRON=0/2 * * * * ? *
JOB_NAME_ADDTASK=base-sync-finance-amazon-addtask
JOB_CRON_ADDTASK=0 0 9 * * ? *
JOB_NAME_GETRESULT=base-sync-finance-amazon-getresult
JOB_CRON_GETRESULT=0 0 6,23 * * ? *
SHARDING_TOTAL_COUNT=3
\ No newline at end of file
data-base/base_sync_finance_amazon/src/main/resources/redis.properties 0 → 100644
#REDIS_URL=94.191.111.219
#REDIS_PORT=6379
REDIS_URL=redis.blt.com
REDIS_PORT=6379
SETMAXTOTAL=100
SETMAXIDLE=10
SETMAXWAITMILLLIS=10000
EXPIRETIME=86400
data-common/src/main/java/com/bailuntec/domain/entity/DcAmazonAdvert.java
......@@ -3,6 +3,7 @@ package com.bailuntec.domain.entity;
import lombok.Data;
import java.math.BigDecimal;
import java.time.LocalDateTime;
@Data
public class DcAmazonAdvert {
......@@ -31,7 +32,7 @@ public class DcAmazonAdvert {
*
* @mbg.generated
*/
private Integer campaignId;
private String campaignId;
/**
*
......@@ -40,7 +41,7 @@ public class DcAmazonAdvert {
*
* @mbg.generated
*/
private Integer campaignStatus;
private String campaignStatus;
/**
*
......@@ -169,6 +170,51 @@ public class DcAmazonAdvert {
private Integer attributedSalesSevenday;
/**
*
* This field was generated by MyBatis Generator.
* This field corresponds to the database column dc_amazon_advert.account_id
*
* @mbg.generated
*/
private Integer accountId;
/**
*
* This field was generated by MyBatis Generator.
* This field corresponds to the database column dc_amazon_advert.company_id
*
* @mbg.generated
*/
private Integer companyId;
/**
*
* This field was generated by MyBatis Generator.
* This field corresponds to the database column dc_amazon_advert.currency
*
* @mbg.generated
*/
private String currency;
/**
*
* This field was generated by MyBatis Generator.
* This field corresponds to the database column dc_amazon_advert.gmt_create_time
*
* @mbg.generated
*/
private LocalDateTime gmtCreateTime;
/**
*
* This field was generated by MyBatis Generator.
* This field corresponds to the database column dc_amazon_advert.gmt_modify_time
*
* @mbg.generated
*/
private LocalDateTime gmtModifyTime;
/**
* This method was generated by MyBatis Generator.
* This method corresponds to the database table dc_amazon_advert
*
......@@ -198,6 +244,11 @@ public class DcAmazonAdvert {
sb.append(", attributedSalesThirtyday=").append(attributedSalesThirtyday);
sb.append(", attributedSalesFourteenday=").append(attributedSalesFourteenday);
sb.append(", attributedSalesSevenday=").append(attributedSalesSevenday);
sb.append(", accountId=").append(accountId);
sb.append(", companyId=").append(companyId);
sb.append(", currency=").append(currency);
sb.append(", gmtCreateTime=").append(gmtCreateTime);
sb.append(", gmtModifyTime=").append(gmtModifyTime);
sb.append("]");
return sb.toString();
}
......@@ -237,7 +288,12 @@ public class DcAmazonAdvert {
&& (this.getAttributedConversionsSevenday() == null ? other.getAttributedConversionsSevenday() == null : this.getAttributedConversionsSevenday().equals(other.getAttributedConversionsSevenday()))
&& (this.getAttributedSalesThirtyday() == null ? other.getAttributedSalesThirtyday() == null : this.getAttributedSalesThirtyday().equals(other.getAttributedSalesThirtyday()))
&& (this.getAttributedSalesFourteenday() == null ? other.getAttributedSalesFourteenday() == null : this.getAttributedSalesFourteenday().equals(other.getAttributedSalesFourteenday()))
&& (this.getAttributedSalesSevenday() == null ? other.getAttributedSalesSevenday() == null : this.getAttributedSalesSevenday().equals(other.getAttributedSalesSevenday()));
&& (this.getAttributedSalesSevenday() == null ? other.getAttributedSalesSevenday() == null : this.getAttributedSalesSevenday().equals(other.getAttributedSalesSevenday()))
&& (this.getAccountId() == null ? other.getAccountId() == null : this.getAccountId().equals(other.getAccountId()))
&& (this.getCompanyId() == null ? other.getCompanyId() == null : this.getCompanyId().equals(other.getCompanyId()))
&& (this.getCurrency() == null ? other.getCurrency() == null : this.getCurrency().equals(other.getCurrency()))
&& (this.getGmtCreateTime() == null ? other.getGmtCreateTime() == null : this.getGmtCreateTime().equals(other.getGmtCreateTime()))
&& (this.getGmtModifyTime() == null ? other.getGmtModifyTime() == null : this.getGmtModifyTime().equals(other.getGmtModifyTime()));
}
/**
......@@ -268,6 +324,11 @@ public class DcAmazonAdvert {
result = prime * result + ((getAttributedSalesThirtyday() == null) ? 0 : getAttributedSalesThirtyday().hashCode());
result = prime * result + ((getAttributedSalesFourteenday() == null) ? 0 : getAttributedSalesFourteenday().hashCode());
result = prime * result + ((getAttributedSalesSevenday() == null) ? 0 : getAttributedSalesSevenday().hashCode());
result = prime * result + ((getAccountId() == null) ? 0 : getAccountId().hashCode());
result = prime * result + ((getCompanyId() == null) ? 0 : getCompanyId().hashCode());
result = prime * result + ((getCurrency() == null) ? 0 : getCurrency().hashCode());
result = prime * result + ((getGmtCreateTime() == null) ? 0 : getGmtCreateTime().hashCode());
result = prime * result + ((getGmtModifyTime() == null) ? 0 : getGmtModifyTime().hashCode());
return result;
}
}
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment