新增抓取Amazon广告费的任务

data-base/base-sync-amazon-ad/Dockerfile

+FROM maven:3.6.0-alpine
+
+RUN apk add ca-certificates && \
+    apk add tzdata && \
+    ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && \
+    echo "Asia/Shanghai" > /etc/timezone
+
+# make source folder
+RUN mkdir -p /usr/app
+WORKDIR /usr/app
+# install maven dependency packages (keep in image)
+
+COPY data-parent /usr/app/data-parent
+COPY data-common /usr/app/data-common
+COPY data-base /usr/app/data-base
+COPY data-mid /usr/app/data-mid
+COPY data-show /usr/app/data-show
+
+RUN cd /usr/app/data-parent && mvn -T 1C install  -pl ../data-base/base-sync-amazon-ad -am
+
+#指定容器启动程序及参数   <ENTRYPOINT> "<CMD>"
+EXPOSE 8080
+ENTRYPOINT ["java","-Xms30m","-Xmx80m","-jar","/usr/app/data-base/base-sync-amazon-ad/target/base-sync-amazon-ad-1.0-SNAPSHOT.jar"]
\ No newline at end of file

data-base/base-sync-amazon-ad/pom.xml

+<?xml version="1.0" encoding="UTF-8"?>
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>data-base</artifactId>
+        <groupId>com.bailuntec</groupId>
+        <version>1.0-SNAPSHOT</version>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+
+    <artifactId>base-sync-amazon-ad</artifactId>
+
+    <name>base-sync-amazon-ad</name>
+    <!-- FIXME change it to the project's website -->
+    <url>http://www.example.com</url>
+
+    <properties>
+        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+        <maven.compiler.source>1.8</maven.compiler.source>
+        <maven.compiler.target>1.8</maven.compiler.target>
+    </properties>
+
+    <dependencies>
+
+        <dependency>
+            <groupId>com.bailuntec</groupId>
+            <artifactId>data-common</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.projectlombok</groupId>
+            <artifactId>lombok</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.junit.jupiter</groupId>
+            <artifactId>junit-jupiter-api</artifactId>
+            <version>RELEASE</version>
+            <scope>test</scope>
+        </dependency>
+
+
+        <dependency>
+            <groupId>redis.clients</groupId>
+            <artifactId>jedis</artifactId>
+            <scope>compile</scope>
+        </dependency>
+
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-compiler-plugin</artifactId>
+            </plugin>
+
+            <plugin>
+                <artifactId>maven-assembly-plugin</artifactId>
+                <version>3.1.0</version>
+                <configuration>
+                    <archive>
+                        <manifest>
+                            <mainClass>com.bailuntec.Application</mainClass>
+                        </manifest>
+                    </archive>
+                    <descriptorRefs>
+                        <descriptorRef>jar-with-dependencies</descriptorRef>
+                    </descriptorRefs>
+                    <appendAssemblyId>false</appendAssemblyId>
+                </configuration>
+                <executions>
+                    <execution>
+                        <id>make-assembly</id>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>single</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+
+        <resources>
+            <resource>
+                <directory>src/main/resources</directory>
+                <includes>
+                    <include>**/*.xml</include>
+                    <include>**/*.properties</include>
+                </includes>
+                <!--默认是true-->
+                <filtering>true</filtering>
+            </resource>
+            <resource>
+                <directory>src/main/java</directory>
+                <includes>
+                    <include>**/*.xml</include>
+                    <include>**/*.properties</include>
+                </includes>
+            </resource>
+        </resources>
+    </build>
+</project>

data-base/base-sync-amazon-ad/src/main/java/com/bailuntec/Application.java

+package com.bailuntec;
+
+import com.alibaba.druid.pool.DruidDataSource;
+import com.bailuntec.job.AmazonAdProductJob;
+import com.bailuntec.listener.AdvertSyncJobListener;
+import com.bailuntec.utils.PropertiesUtil;
+import com.dangdang.ddframe.job.config.JobCoreConfiguration;
+import com.dangdang.ddframe.job.config.simple.SimpleJobConfiguration;
+import com.dangdang.ddframe.job.event.JobEventConfiguration;
+import com.dangdang.ddframe.job.event.rdb.JobEventRdbConfiguration;
+import com.dangdang.ddframe.job.lite.api.JobScheduler;
+import com.dangdang.ddframe.job.lite.config.LiteJobConfiguration;
+import com.dangdang.ddframe.job.reg.base.CoordinatorRegistryCenter;
+import com.dangdang.ddframe.job.reg.zookeeper.ZookeeperConfiguration;
+import com.dangdang.ddframe.job.reg.zookeeper.ZookeeperRegistryCenter;
+import lombok.extern.slf4j.Slf4j;
+
+import javax.sql.DataSource;
+
+@Slf4j
+public class Application {
+    private static PropertiesUtil propertiesUtil = PropertiesUtil.getInstance("job");
+    private static final String EVENT_RDB_STORAGE_DRIVER = propertiesUtil.getPropertyAsString("EVENT_RDB_STORAGE_DRIVER");
+    private static final String EVENT_RDB_STORAGE_URL = propertiesUtil.getPropertyAsString("EVENT_RDB_STORAGE_URL");
+    private static final String EVENT_RDB_STORAGE_USERNAME = propertiesUtil.getPropertyAsString("EVENT_RDB_STORAGE_USERNAME");
+    private static final String EVENT_RDB_STORAGE_PASSWORD = propertiesUtil.getPropertyAsString("EVENT_RDB_STORAGE_PASSWORD");
+
+    public static void main(String[] args) {
+        new JobScheduler(createRegistryCenter(), createJobConfiguration1(), createJobEventConfiguration(), new AdvertSyncJobListener()).init();
+    }
+
+    private static CoordinatorRegistryCenter createRegistryCenter() {
+        CoordinatorRegistryCenter regCenter = new ZookeeperRegistryCenter(new ZookeeperConfiguration(propertiesUtil.getPropertyAsString("ZOOKEEPER_SERVER"), propertiesUtil.getPropertyAsString("NAME_SPACE")));
+        regCenter.init();
+        return regCenter;
+    }
+
+    private static LiteJobConfiguration createJobConfiguration1() {
+        JobCoreConfiguration simpleCoreConfig = JobCoreConfiguration.newBuilder(propertiesUtil.getPropertyAsString("JOB_NAME_AD_PRODUCT"), propertiesUtil.getPropertyAsString("JOB_CRON_AD_PRODUCT"), propertiesUtil.getPropertyAsInt("SHARDING_TOTAL_COUNT")).build();
+        SimpleJobConfiguration simpleJobConfig = new SimpleJobConfiguration(simpleCoreConfig, AmazonAdProductJob.class.getCanonicalName());
+        LiteJobConfiguration simpleJobRootConfig = LiteJobConfiguration.newBuilder(simpleJobConfig).build();
+        return simpleJobRootConfig;
+    }
+
+    private static JobEventConfiguration createJobEventConfiguration() {
+        JobEventConfiguration jobEventRdbConfig = new JobEventRdbConfiguration(setUpEventTraceDataSource());
+        return jobEventRdbConfig;
+    }
+
+    private static DataSource setUpEventTraceDataSource() {
+        DruidDataSource result = new DruidDataSource();
+        result.setDriverClassName(EVENT_RDB_STORAGE_DRIVER);
+        result.setUrl(EVENT_RDB_STORAGE_URL);
+        result.setUsername(EVENT_RDB_STORAGE_USERNAME);
+        result.setPassword(EVENT_RDB_STORAGE_PASSWORD);
+        return result;
+    }
+
+}

data-base/base-sync-amazon-ad/src/main/java/com/bailuntec/domain/pojo/AmazonAdAuth.java

+package com.bailuntec.domain.pojo;
+
+import com.alibaba.fastjson.annotation.JSONField;
+import lombok.Data;
+
+@Data
+public class AmazonAdAuth {
+    @JSONField(name = "access_token")
+    private String accessToken;
+    @JSONField(name = "refresh_token")
+    private String refreshToken;
+    @JSONField(name = "token_type")
+    private String tokenType;
+    @JSONField(name = "profile_id")
+    private String profileId;
+    @JSONField(name = "client_id")
+    private String clientId;
+}

data-base/base-sync-amazon-ad/src/main/java/com/bailuntec/domain/pojo/AmazonAdReportLocationResult.java

+package com.bailuntec.domain.pojo;
+
+import lombok.Data;
+
+@Data
+public class AmazonAdReportLocationResult {
+    private String reportId;
+    private String status;
+    private String statusDetails;
+    private String location;
+    private long fileSize;
+}

data-base/base-sync-amazon-ad/src/main/java/com/bailuntec/domain/pojo/AmazonAdSuccessResult.java

+package com.bailuntec.domain.pojo;
+
+import lombok.Data;
+
+@Data
+public class AmazonAdSuccessResult {
+    private String reportId;
+    private String recordType;
+    private String status;
+    private String statusDetails;
+}

data-base/base-sync-amazon-ad/src/main/java/com/bailuntec/listener/AdvertSyncJobListener.java

+package com.bailuntec.listener;
+
+import com.dangdang.ddframe.job.executor.ShardingContexts;
+import com.dangdang.ddframe.job.lite.api.listener.ElasticJobListener;
+
+public class AdvertSyncJobListener implements ElasticJobListener {
+    @Override
+    public void beforeJobExecuted(ShardingContexts shardingContexts) {
+
+    }
+    @Override
+    public void afterJobExecuted(ShardingContexts shardingContexts) {
+
+    }
+}

data-base/base-sync-amazon-ad/src/main/java/com/bailuntec/utils/RedisUtils.java

+package com.bailuntec.utils;
+
+import lombok.extern.slf4j.Slf4j;
+import redis.clients.jedis.Jedis;
+import redis.clients.jedis.JedisPool;
+import redis.clients.jedis.JedisPoolConfig;
+
+@Slf4j
+public class RedisUtils {
+
+    private static PropertiesUtil propertiesUtil = PropertiesUtil.getInstance("redis");
+    private static JedisPool jedisPool = null;
+
+    /*
+    *  初始化连接池
+     */
+    static{
+        JedisPoolConfig config = new JedisPoolConfig();
+        config.setMaxTotal(propertiesUtil.getPropertyAsInt("SETMAXTOTAL"));
+        config.setMaxIdle(propertiesUtil.getPropertyAsInt("SETMAXIDLE"));
+        config.setMaxWaitMillis(propertiesUtil.getPropertyAsInt("SETMAXWAITMILLLIS"));
+        config.setTestOnBorrow(true);
+        jedisPool = new JedisPool(config, propertiesUtil.getPropertyAsString("REDIS_URL"), propertiesUtil.getPropertyAsInt("REDIS_PORT"));
+        log.warn("初始化连接池");
+    }
+
+    /*
+    *  获取redis实例
+     */
+    public synchronized static Jedis getJedis() {
+        Jedis jedis = null;
+        try{
+            jedis =jedisPool.getResource();
+        }catch(Exception e) {
+            e.printStackTrace();
+        }
+        return jedis;
+    }
+
+    /*
+    *  回收redis对象资源
+     */
+    public synchronized static void returnResource(Jedis jedis) {
+        if(jedis != null) {
+            try{
+                jedis.close();
+            }catch (Exception e){
+                e.printStackTrace();
+            }
+        }
+    }
+
+    /*
+    *  获取key值
+     */
+    public String getValueByKey(String key) {
+        Jedis jedis = null;
+        String result = null;
+        try {
+            jedis = getJedis();
+            result = jedis.get(key);
+        } catch (Exception e) {
+            e.printStackTrace();
+        } finally {
+            returnResource(jedis);
+        }
+        return result;
+    }
+
+    /*
+    *  删除key
+     */
+    public void deleteByKey(String key) throws Exception {
+        Jedis jedis = null;
+        try {
+            jedis = getJedis();
+            jedis.del(key);
+        } catch (Exception e) {
+            throw e;
+        } finally {
+            returnResource(jedis);
+        }
+    }
+
+    /*
+    *  设置过期时间值
+     */
+    public void saveValueByKey(String key, String value) throws Exception {
+        Jedis jedis = null;
+        try {
+            jedis = getJedis();
+            jedis.set(key, value);
+            jedis.expire(key, propertiesUtil.getPropertyAsInt("EXPIRETIME"));
+        } catch (Exception e) {
+            throw e;
+        } finally {
+            returnResource(jedis);
+        }
+    }
+}

data-base/base-sync-amazon-ad/src/main/resources/const.properties

+NA_URL=https://advertising-api.amazon.com
+EU_URL=https://advertising-api-eu.amazon.com
+FE_URL=https://advertising-api-fe.amazon.com
+POST_PRODUCT_ADS_REPORT_ID=/v2/sp/productAds/report
+GET_PRODUCT_ADS_REPORT_LOCATION=/v2/reports/
+WARN_API=https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=b4655a50-927a-4c3c-87ad-a900e54dcc8e
+METRICS=campaignName,campaignId,adGroupName,adGroupId,impressions,clicks,cost,currency,asin,sku,attributedConversions1d,attributedConversions7d,attributedConversions14d,attributedConversions30d,attributedConversions1dSameSKU,attributedConversions7dSameSKU,attributedConversions14dSameSKU,attributedConversions30dSameSKU,attributedUnitsOrdered1d,attributedUnitsOrdered7d,attributedUnitsOrdered14d,attributedUnitsOrdered30d,attributedSales1d,attributedSales7d,attributedSales14d,attributedSales30d,attributedSales1dSameSKU,attributedSales7dSameSKU,attributedSales14dSameSKU,attributedSales30dSameSKU,attributedUnitsOrdered1dSameSKU,attributedUnitsOrdered7dSameSKU,attributedUnitsOrdered14dSameSKU,attributedUnitsOrdered30dSameSKU
+

data-base/base-sync-amazon-ad/src/main/resources/job.properties

+#EVENT_RDB_STORAGE_DRIVER=com.mysql.jdbc.Driver
+#EVENT_RDB_STORAGE_URL=jdbc:mysql://127.0.0.1:3306/bailun_datacenter?useUnicode=true&characterEncoding=UTF-8&serverTimeZone=GMT&useSSL=false&allowPublicKeyRetrieval=true&allowMultiQueries=true
+#EVENT_RDB_STORAGE_USERNAME=root
+#EVENT_RDB_STORAGE_PASSWORD=root
+#ZOOKEEPER_SERVER=94.191.111.219:2181
+EVENT_RDB_STORAGE_DRIVER=com.mysql.jdbc.Driver
+EVENT_RDB_STORAGE_URL=jdbc:mysql://10.0.8.15:3306/bailun_datacenter?useUnicode=true&characterEncoding=utf-8&serverTimezone=GMT%2B8
+EVENT_RDB_STORAGE_USERNAME=root
+EVENT_RDB_STORAGE_PASSWORD=#7kfnymAM$Y9-Ntf
+ZOOKEEPER_SERVER=172.31.255.120:2181
+NAME_SPACE=data-center
+JOB_NAME_AD_PRODUCT=base-sync-finance-amazon-ad-product
+JOB_CRON_AD_PRODUCT=0/10 * * * * * *
+SHARDING_TOTAL_COUNT=1
\ No newline at end of file

data-base/base-sync-amazon-ad/src/main/resources/redis.properties

+REDIS_URL=94.191.111.219
+REDIS_PORT=6379
+
+#REDIS_URL=redis.blt.com
+#REDIS_PORT=6379
+
+SETMAXTOTAL=100
+SETMAXIDLE=10
+SETMAXWAITMILLLIS=10000
+EXPIRETIME=172800
+
+
+

data-base/base-sync-amazon-ad/src/test/java/AmazonAdProductTest.java

+import com.bailuntec.domain.entity.DcBaseCompanyAccount;
+import com.bailuntec.domain.entity.JobAccountLog;
+import com.bailuntec.domain.entity.JobPointLog;
+import com.bailuntec.domain.example.DcBaseCompanyAccountExample;
+import com.bailuntec.domain.example.JobAccountLogExample;
+import com.bailuntec.job.AmazonAdProductJob;
+import com.bailuntec.mapper.DcBaseCompanyAccountMapper;
+import com.bailuntec.mapper.JobAccountLogMapper;
+import com.bailuntec.utils.SessionUtil;
+import com.dangdang.ddframe.job.api.ShardingContext;
+import com.dangdang.ddframe.job.executor.ShardingContexts;
+import org.junit.jupiter.api.Test;
+
+import java.time.LocalDateTime;
+import java.util.HashMap;
+
+public class AmazonAdProductTest {
+
+    @Test
+    public void test() {
+        AmazonAdProductJob amazonAdProductJob = new AmazonAdProductJob();
+        ShardingContexts shardingContexts = new ShardingContexts(null,"base-sync-finance-amazon-ad-product",1,null,new HashMap<>());
+        ShardingContext shardingContext = new ShardingContext(shardingContexts,0);
+        JobPointLog jobPointLog = new JobPointLog("base-sync-finance-amazon-ad-product", 1, 100, 0, 1,
+                LocalDateTime.of(2019, 7, 1, 0, 0),
+                LocalDateTime.of(2019, 7, 2, 0, 0));
+        amazonAdProductJob.executeJob(shardingContext, jobPointLog);
+    }
+    @Test
+    public void test1() {
+        AmazonAdProductJob amazonAdProductJob = new AmazonAdProductJob();
+        JobAccountLogMapper mapper = SessionUtil.getSession().getMapper(JobAccountLogMapper.class);
+        JobAccountLog dcBaseCompanyAccount = mapper.selectOneByExample(JobAccountLogExample.newAndCreateCriteria().andAccountIdEqualTo(591).example());
+        amazonAdProductJob.callAmazonAdApi(dcBaseCompanyAccount);
+    }
+}

data-base/pom.xml

@@ -30,6 +30,7 @@
         <module>base-sync-finance-ebay</module>
         <module>base-sync-paypal</module>
         <module>base-sync-finance-amazon</module>
+        <module>base-sync-amazon-ad</module>
     </modules>
 
     <packaging>pom</packaging>

data-common/src/main/java/com/bailuntec/mapper/DcBaseFinanceAmazonAdProductMapper.java

+package com.bailuntec.mapper;
+
+import com.bailuntec.domain.entity.DcBaseFinanceAmazonAdProduct;
+import com.bailuntec.domain.example.DcBaseFinanceAmazonAdProductExample;
+import org.apache.ibatis.annotations.Param;
+
+import java.util.List;
+
+public interface DcBaseFinanceAmazonAdProductMapper {
+    /**
+     * This method was generated by MyBatis Generator.
+     * This method corresponds to the database table dc_base_finance_amazon_ad_product
+     *
+     * @mbg.generated
+     */
+    long countByExample(DcBaseFinanceAmazonAdProductExample example);
+
+    /**
+     * This method was generated by MyBatis Generator.
+     * This method corresponds to the database table dc_base_finance_amazon_ad_product
+     *
+     * @mbg.generated
+     */
+    int deleteByExample(DcBaseFinanceAmazonAdProductExample example);
+
+    /**
+     * This method was generated by MyBatis Generator.
+     * This method corresponds to the database table dc_base_finance_amazon_ad_product
+     *
+     * @mbg.generated
+     */
+    int deleteByPrimaryKey(Integer id);
+
+    /**
+     * This method was generated by MyBatis Generator.
+     * This method corresponds to the database table dc_base_finance_amazon_ad_product
+     *
+     * @mbg.generated
+     */
+    int insert(DcBaseFinanceAmazonAdProduct record);
+
+    /**
+     * This method was generated by MyBatis Generator.
+     * This method corresponds to the database table dc_base_finance_amazon_ad_product
+     *
+     * @mbg.generated
+     */
+    int insertSelective(DcBaseFinanceAmazonAdProduct record);
+
+    /**
+     * This method was generated by MyBatis Generator.
+     * This method corresponds to the database table dc_base_finance_amazon_ad_product
+     *
+     * @mbg.generated
+     * @project https://github.com/itfsw/mybatis-generator-plugin
+     */
+    DcBaseFinanceAmazonAdProduct selectOneByExample(DcBaseFinanceAmazonAdProductExample example);
+
+    /**
+     * This method was generated by MyBatis Generator.
+     * This method corresponds to the database table dc_base_finance_amazon_ad_product
+     *
+     * @mbg.generated
+     */
+    List<DcBaseFinanceAmazonAdProduct> selectByExample(DcBaseFinanceAmazonAdProductExample example);
+
+    /**
+     * This method was generated by MyBatis Generator.
+     * This method corresponds to the database table dc_base_finance_amazon_ad_product
+     *
+     * @mbg.generated
+     */
+    DcBaseFinanceAmazonAdProduct selectByPrimaryKey(Integer id);
+
+    /**
+     * This method was generated by MyBatis Generator.
+     * This method corresponds to the database table dc_base_finance_amazon_ad_product
+     *
+     * @mbg.generated
+     */
+    int updateByExampleSelective(@Param("record") DcBaseFinanceAmazonAdProduct record, @Param("example") DcBaseFinanceAmazonAdProductExample example);
+
+    /**
+     * This method was generated by MyBatis Generator.
+     * This method corresponds to the database table dc_base_finance_amazon_ad_product
+     *
+     * @mbg.generated
+     */
+    int updateByExample(@Param("record") DcBaseFinanceAmazonAdProduct record, @Param("example") DcBaseFinanceAmazonAdProductExample example);
+
+    /**
+     * This method was generated by MyBatis Generator.
+     * This method corresponds to the database table dc_base_finance_amazon_ad_product
+     *
+     * @mbg.generated
+     */
+    int updateByPrimaryKeySelective(DcBaseFinanceAmazonAdProduct record);
+
+    /**
+     * This method was generated by MyBatis Generator.
+     * This method corresponds to the database table dc_base_finance_amazon_ad_product
+     *
+     * @mbg.generated
+     */
+    int updateByPrimaryKey(DcBaseFinanceAmazonAdProduct record);
+
+    /**
+     * This method was generated by MyBatis Generator.
+     * This method corresponds to the database table dc_base_finance_amazon_ad_product
+     *
+     * @mbg.generated
+     * @project https://github.com/itfsw/mybatis-generator-plugin
+     */
+    int upsert(DcBaseFinanceAmazonAdProduct record);
+
+    /**
+     * This method was generated by MyBatis Generator.
+     * This method corresponds to the database table dc_base_finance_amazon_ad_product
+     *
+     * @mbg.generated
+     * @project https://github.com/itfsw/mybatis-generator-plugin
+     */
+    int upsertSelective(DcBaseFinanceAmazonAdProduct record);
+}
\ No newline at end of file

data-common/src/main/java/com/bailuntec/utils/ExceptionUtil.java

@@ -18,7 +18,7 @@ public class ExceptionUtil {
         try (PrintWriter writer = new PrintWriter(result)) {
             cause.printStackTrace(writer);
         }
-        return result.toString();
+        return result.toString().length() > 3500?result.toString().substring(0,3500) : result.toString();
     }
 
 

data-common/src/main/resources/mybatis-config.xml

@@ -4,8 +4,8 @@
         "http://mybatis.org/dtd/mybatis-3-config.dtd">
 <configuration>
     <!--<properties  resource="db-tj.properties"/>-->
-    <properties  resource="db-dev.properties"/>
-<!--    <properties  resource="db-prod.properties"/>-->
+<!--    <properties  resource="db-dev.properties"/>-->
+    <properties  resource="db-prod.properties"/>
     <settings>
         <setting name="mapUnderscoreToCamelCase" value="true"/>
     </settings>