amazon广告费抓取

data-base/base_sync_finance_amazon/src/main/java/com/bailuntec/Application.java

@@ -27,7 +27,7 @@ public class Application {
 
     public static void main(String[] args) {
         new JobScheduler(createRegistryCenter(), createJobConfiguration1()).init();
-        new JobScheduler(createRegistryCenter(), createJobConfiguration2()).init();
+      //  new JobScheduler(createRegistryCenter(), createJobConfiguration2()).init();
     }
 
     private static CoordinatorRegistryCenter createRegistryCenter() {

data-base/base_sync_finance_amazon/src/main/java/com/bailuntec/domain/AccountTokenInfo.java

+package com.bailuntec.domain;
+
+import lombok.Data;
+
+import java.util.List;
+import java.util.Map;
+
+@Data
+public class AccountTokenInfo {
+
+    private Boolean success;
+    private String message;
+    private List<Map> data;
+
+}

data-base/base_sync_finance_amazon/src/main/java/com/bailuntec/job/AdvertAddTaskSyncJob.java

 package com.bailuntec.job;
 
 import com.alibaba.fastjson.JSON;
-import com.alibaba.fastjson.JSONObject;
-import com.bailuntec.domain.AdvertResultInfo;
-import com.bailuntec.domain.entity.JobAccountLog;
+import com.bailuntec.domain.AccountTokenInfo;
+import com.bailuntec.domain.AmazonAdvert;
 import com.bailuntec.domain.entity.JobPointLog;
-import com.bailuntec.domain.enumerate.AccountPlatformType;
-import com.bailuntec.domain.example.DcBaseCompanyAccountExample;
-import com.bailuntec.domain.example.JobAccountLogExample;
-import com.bailuntec.mapper.DcBaseCompanyAccountMapper;
-import com.bailuntec.mapper.JobAccountLogMapper;
+import com.bailuntec.mapper.DcBaseFinanceAmazonMapper;
 import com.bailuntec.support.PointJob;
 import com.bailuntec.utils.OkHttpUtil;
 import com.bailuntec.utils.PropertiesUtil;
@@ -17,13 +12,15 @@ import com.bailuntec.utils.SessionUtil;
 import com.dangdang.ddframe.job.api.ShardingContext;
 import lombok.extern.slf4j.Slf4j;
 import okhttp3.*;
-import org.apache.commons.beanutils.BeanUtils;
-
+import org.apache.commons.lang3.StringUtils;
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
 import java.time.LocalDateTime;
 import java.time.format.DateTimeFormatter;
-import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
+import java.util.zip.GZIPInputStream;
 
 @Slf4j
 public class AdvertAddTaskSyncJob extends PointJob {
@@ -33,103 +30,103 @@ public class AdvertAddTaskSyncJob extends PointJob {
 
     @Override
     public void executeJob(ShardingContext shardingContext, JobPointLog jobPointLog) {
-        int totalPage = getCount(jobPointLog);//分片即将处理的账号总数
-        try {
-            JobAccountLogMapper jobAccountLogMapper = SessionUtil.getSession().getMapper(JobAccountLogMapper.class);
-            int pageSize = totalPage % shardingContext.getShardingTotalCount() == 0 ? totalPage / shardingContext.getShardingTotalCount() : totalPage / shardingContext.getShardingTotalCount() + 1;
-            List<JobAccountLog> listByPage = jobAccountLogMapper.getListByPage(AccountPlatformType.Amazon.code(), pageSize * shardingContext.getShardingItem(), pageSize);
-            if (listByPage != null && listByPage.size() > 0) {
-                for (JobAccountLog jobAccountLog : listByPage) {
-                    log.warn("分片: " + shardingContext.getShardingItem() + " 账号: " + jobAccountLog);
-                    if (jobAccountLog.getId() == null) { //在任务表无记录
-                        BeanUtils.copyProperties(jobAccountLog, jobPointLog);
-                        jobAccountLog.setId(null);
-                    }
-                    callAddTask(jobAccountLog);
-                    if (jobAccountLog.getId() == null) { //在任务表无记录
-                        jobAccountLogMapper.insertSelective(jobAccountLog);
-                    } else {
-                        jobAccountLog.setBjModified(LocalDateTime.now());
-                        jobAccountLogMapper.updateByExampleSelective(jobAccountLog, JobAccountLogExample.newAndCreateCriteria().andAccountIdEqualTo(jobAccountLog.getAccountId()).andJobNameEqualTo(jobAccountLog.getJobName()).example());
-                    }
-                }
-            }
-        } catch (Exception e) {
-            e.printStackTrace();
-            log.warn("Amazon任务发布接口调用错误", e);
-        } finally {
-            SessionUtil.closeSession();
-        }
+        callAddTask(jobPointLog);
     }
 
-    private int getCount(JobPointLog jobPointLog) {
-        int i = 0;
-        try {
-            DcBaseCompanyAccountMapper accountMapper = SessionUtil.getSession().getMapper(DcBaseCompanyAccountMapper.class);
-            i = (int) accountMapper.countByExample(DcBaseCompanyAccountExample.newAndCreateCriteria().andPlatformIdEqualTo(AccountPlatformType.Amazon.code()).example());
-        } catch (Exception e) {
-            e.printStackTrace();
-        } finally {
-            SessionUtil.closeSession();
-        }
-        return i;
-    }
-
-    public void callAddTask(JobAccountLog job) {
+    public void callAddTask(JobPointLog job) {
         LocalDateTime localDateTime = job.getStartTime();
-        String resultStr = "";
-        String str = "[\"campaignName\", \"campaignId\", \"campaignStatus\", \"campaignBudget\", \"attributedConversions1d\", \"attributedUnitsOrdered1d\", \"attributedSales1d\", \"attributedUnitsOrdered7d\", \"attributedSales1d\", \"impressions\", \"clicks\", \"cost\", \"attributedConversions30d\", \"attributedConversions14d\", \"attributedConversions7d\", \"attributedSales30d\", \"attributedSales14d\", \"attributedSales7d\"]";
-        HashMap<String, Object> map = new HashMap<>();
-        List listTasks = new ArrayList();
-        HashMap<String, Object> mapTasks = new HashMap<>();
-        HashMap<String, Object> mapTaskInfo = new HashMap<>();
-        List listMetrics = JSON.parseObject(str, List.class);
-        mapTaskInfo.put("type", "sp");
-        mapTaskInfo.put("record_type", "campaigns");
-        mapTaskInfo.put("segment", null);
-        mapTaskInfo.put("metrics", listMetrics);
-        mapTasks.put("account_id", job.getAccountId());
-        mapTasks.put("task_type", "amazon:advertising:report");
-        mapTasks.put("task_info", mapTaskInfo);
-        map.put("app_id", 1);
-        listTasks.add(mapTasks);
-        map.put("tasks", listTasks);
-
         MediaType mediaType = MediaType.parse("application/json");
         Response response = null;
-        List<List<Integer>> taskIdsLists = new ArrayList<>();
         try {
             do {
-                mapTaskInfo.put("report_date", localDateTime.format(DateTimeFormatter.ofPattern("yyyyMMdd")));
-                RequestBody body = RequestBody.create(mediaType, JSON.toJSONString(map));
-
-                Request request = new Request.Builder()
-                        .url(propertiesUtil.getPropertyAsString("ADDTASK_FINANCE_URL"))
-                        .post(body)
+                Request request0 = new Request.Builder()
+                        .url(propertiesUtil.getPropertyAsString("ACCOUNT_API"))
                         .addHeader("Content-Type", "application/json")
                         .build();
-                response = client.newCall(request).execute();
-                AdvertResultInfo advertResult = JSON.parseObject(response.body().string(), AdvertResultInfo.class);
-                if (advertResult != null && advertResult.getCode() == 0) {
-                    resultStr = advertResult.getTaskIds();
-                    List<Integer> idsLists = JSON.parseArray(resultStr, Integer.class);
-                    taskIdsLists.add(idsLists);
+                response = client.newCall(request0).execute();
+                AccountTokenInfo resultMap0 = JSON.parseObject(response.body().string(), AccountTokenInfo.class);
+                List<Map> list = resultMap0.getData();
+                Map<String, String> authJsonMap = new HashMap<String, String>();
+                if(list != null && list.size() > 0) {
+                    for(Map map0 : list) {
+                        String authStr = ((String) map0.get("AmazonAdAuthJson"));
+                        if(StringUtils.isBlank(authStr)){
+                            continue;
+                        }
+                        authJsonMap = JSON.parseObject(authStr, Map.class);
+
+                        Map<String, String> map = new HashMap<>();
+                        map.put("reportDate", localDateTime.format(DateTimeFormatter.ofPattern("yyyyMMdd")));
+                        map.put("metrics", propertiesUtil.getPropertyAsString("METRICS"));
+                        RequestBody body = RequestBody.create(mediaType, JSON.toJSONString(map));
+                        Request request = new Request.Builder()
+                                .url(propertiesUtil.getPropertyAsString("CAMPAIGNS_REPORT"))
+                                .post(body)
+                                .addHeader("Authorization", "Bearer " + authJsonMap.get("access_token"))
+                                .addHeader("Amazon-Advertising-API-ClientId", authJsonMap.get("client_id"))
+                                .addHeader("Amazon-Advertising-API-Scope", authJsonMap.get("profile_id"))
+                                .addHeader("Content-Type", "application/json")
+                                .build();
+                        response = client.newCall(request).execute();
+                        Map<String, String> resultMap = JSON.parseObject(response.body().string(), Map.class);
+                        if("IN_PROGRESS".equals(resultMap.get("status"))) {
+                            Request request2 = new Request.Builder()
+                                    .url(propertiesUtil.getPropertyAsString("V2_REPORTS") + resultMap.get("reportId"))
+                                    .addHeader("Authorization", "Bearer " + authJsonMap.get("access_token"))
+                                    .addHeader("Amazon-Advertising-API-ClientId", authJsonMap.get("client_id"))
+                                    .addHeader("Amazon-Advertising-API-Scope", authJsonMap.get("profile_id"))
+                                    .addHeader("Content-Type", "application/json")
+                                    .build();
+                            response = client.newCall(request2).execute();
+                            Map<String, String> resultMap2 = JSON.parseObject(response.body().string(), Map.class);
+                            if("SUCCESS".equals(resultMap2.get("status"))) {
+                                String location = resultMap2.get("location");
+                                Request request3 = new Request.Builder()
+                                        .url(location)
+                                        .addHeader("Authorization", "Bearer " + authJsonMap.get("access_token"))
+                                        .addHeader("Amazon-Advertising-API-ClientId", authJsonMap.get("client_id"))
+                                        .addHeader("Amazon-Advertising-API-Scope", authJsonMap.get("profile_id"))
+                                        .addHeader("Content-Type", "application/json")
+                                        .build();
+                                response = client.newCall(request3).execute();
+                                GZIPInputStream gzin = new GZIPInputStream(response.body().byteStream());
+                                InputStreamReader isr = new InputStreamReader(gzin, "utf8");
+                                BufferedReader bufferedReader = new BufferedReader(isr);
+                                String str = null;
+                                StringBuffer sb = new StringBuffer();
+                                while((str = bufferedReader.readLine()) != null){
+                                    sb.append(str);
+                                }
+                                List<AmazonAdvert> amazonAdverts = JSON.parseArray(sb.toString(), AmazonAdvert.class);
+                                for(AmazonAdvert amazonAdvert : amazonAdverts) {
+                                    amazonAdvert.setAccountId((Integer) map0.get("Id"));
+                                    // amazonAdvert.setCompanyId(job.getCompanyId());
+                                    try {
+                                        DcBaseFinanceAmazonMapper dcBaseFinanceAmazonMapper = SessionUtil.getSession().getMapper(DcBaseFinanceAmazonMapper.class);
+                                        dcBaseFinanceAmazonMapper.upsertSelective(amazonAdvert);
+                                    } catch (Exception e) {
+                                        e.printStackTrace();
+                                        throw new RuntimeException("Mybatis操作DB插入任务记录失败", e);
+                                    } finally {
+                                        SessionUtil.closeSession();
+                                    }
+                                }
+                                gzin.close();
+                                isr.close();
+                            }
+                        }
+                    }
                 }
                 localDateTime = localDateTime.plusDays(1L);
-            } while (localDateTime.compareTo(job.getEndTime()) <= 0);
-            String taskIdsStr = JSONObject.toJSONString(taskIdsLists);
-            job.setTaskIds(taskIdsStr);
+            } while (!localDateTime.isAfter(job.getEndTime()));
             job.setStartTime(job.getEndTime());
             job.setEndTime(job.getStartTime().plusDays(job.getIntervalTime()).isAfter(LocalDateTime.now()) ? LocalDateTime.now() : job.getEndTime().plusDays(job.getIntervalTime()));
-            job.setMessage("执行成功");
         } catch (Exception e) {
-            job.setMessage(e.getMessage());
+            throw new RuntimeException( e);
         } finally {
             if (response != null) {
                 response.close();
             }
         }
     }
-
-
 }

data-base/base_sync_finance_amazon/src/main/resources/const.properties

-#ADDTASK_FINANCE_URL=http://111.230.84.223:18321/outer/addtask
-#GETRESULT_FINANCE_URL=http://111.230.84.223:18321/outer/getresult
 ADDTASK_FINANCE_URL=http://10.0.3.5:18321/outer/addtask
 GETRESULT_FINANCE_URL=http://10.0.3.5:18321/outer/getresult
+ACCOUNT_API=http://pams.bailuntec.com/Api/GetAccountToken?platform=Amazon
+CAMPAIGNS_REPORT=https://advertising-api.amazon.com/v2/sp/campaigns/report
+V2_REPORTS=https://advertising-api.amazon.com/v2/reports/
+#METRICS=campaignName,campaignId,campaignStatus,campaignBudget,attributedConversions1d,attributedUnitsOrdered1d,attributedSales1d,attributedUnitsOrdered7d,attributedSales1d,impressions,clicks,cost,attributedConversions30d,attributedConversions14d,attributedConversions7d,attributedSales30d,attributedSales14d,attributedSales7d
+METRICS=campaignName,campaignId,cost,currency,campaignStatus,campaignBudget,cost
 
 

data-base/base_sync_finance_amazon/src/main/resources/job.properties

@@ -13,4 +13,4 @@ JOB_NAME_ADDTASK=base-sync-finance-amazon-addtask
 JOB_CRON_ADDTASK=0 0 9 * * ? *
 JOB_NAME_GETRESULT=base-sync-finance-amazon-getresult
 JOB_CRON_GETRESULT=0 0 6,23 * * ? *
-SHARDING_TOTAL_COUNT=3
+SHARDING_TOTAL_COUNT=1
\ No newline at end of file