首页手记 Java 数据分批调用接口的正确姿势

Java 数据分批调用接口的正确姿势

标签：

Java

一、背景

现实业务开发中，通常为了避免超时、对方接口限制等原因需要对支持批量的接口的数据分批调用。

比如List参数的size可能为几十个甚至上百个，但是假如对方dubbo接口比较慢，传入50个以上会超时，那么可以每次传入20个，分批执行。

通常很多人会写 for 循环或者 while 循环，非常不优雅，无法复用，而且容易出错。

下面结合 Java8 的 Stream ，Function ,Consumer 等特性实现分批调用的工具类封装和自测。

并给出 CompletableFuture 的异步改进方案。

二、实现

工具类：

package com.chujianyun.common.java8.function;


import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import org.apache.commons.collections4.CollectionUtils;

import java.util.*;
import java.util.function.Consumer;
import java.util.function.Function;

/**
 * 执行工具类
 *
 * @author 明明如月
 */
public class ExecuteUtil {


    public static <T> void partitionRun(List<T> dataList, int size, Consumer<List<T>> consumer) {
        if (CollectionUtils.isEmpty(dataList)) {
            return;
        }
        Preconditions.checkArgument(size > 0, "size must not be a minus");
        Lists.partition(dataList, size).forEach(consumer);
    }

    public static <T, V> List<V> partitionCall2List(List<T> dataList, int size, Function<List<T>, List<V>> function) {

        if (CollectionUtils.isEmpty(dataList)) {
            return new ArrayList<>(0);
        }
        Preconditions.checkArgument(size > 0, "size must not be a minus");

        return Lists.partition(dataList, size)
                .stream()
                .map(function)
                .filter(Objects::nonNull)
                .reduce(new ArrayList<>(),
                        (resultList1, resultList2) -> {
                            resultList1.addAll(resultList2);
                            return resultList1;
                        });


    }

    public static <T, V> Map<T, V> partitionCall2Map(List<T> dataList, int size, Function<List<T>, Map<T, V>> function) {
        if (CollectionUtils.isEmpty(dataList)) {
            return new HashMap<>(0);
        }
        Preconditions.checkArgument(size > 0, "size must not be a minus");
        return Lists.partition(dataList, size)
                .stream()
                .map(function)
                .filter(Objects::nonNull)
                .reduce(new HashMap<>(),
                        (resultMap1, resultMap2) -> {
                            resultMap1.putAll(resultMap2);
                            return resultMap1;
                        });


    }
}

待调用的服务（模拟）

package com.chujianyun.common.java8.function;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

public class SomeManager {

    public void aRun(Long id, List<String> data) {

    }

    public List<Integer> aListMethod(Long id, List<String> data) {
        return new ArrayList<>(0);
    }

    public Map<String, Integer> aMapMethod(Long id, List<String> data) {
        return new HashMap<>(0);
    }
}

单元测试：

package com.chujianyun.common.java8.function;

import org.apache.commons.lang3.RandomUtils;
import org.jeasy.random.EasyRandom;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.internal.verification.Times;
import org.powermock.api.mockito.PowerMockito;
import org.powermock.modules.junit4.PowerMockRunner;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import java.util.stream.Stream;

import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyLong;

@RunWith(PowerMockRunner.class)
public class ExecuteUtilTest {

    private EasyRandom easyRandom = new EasyRandom();

    @Mock
    private SomeManager someManager;

    // 测试数据
    private List<String> mockDataList;

    private int total = 30;

    @Before
    public void init() {
        // 构造30条数据
        mockDataList = easyRandom.objects(String.class, 30).collect(Collectors.toList());

    }

    @Test
    public void test_a_run_partition() {
        // mock aRun
        PowerMockito.doNothing().when(someManager).aRun(anyLong(), any());

        // 每批 10 个
        ExecuteUtil.partitionRun(mockDataList, 10, (eachList) -> someManager.aRun(1L, eachList));

        //验证执行了 3 次
        Mockito.verify(someManager, new Times(3)).aRun(anyLong(), any());
    }


    @Test
    public void test_call_return_list_partition() {
        // mock  每次调用返回条数(注意每次调用都是这2个)
        int eachReturnSize = 2;
        PowerMockito
                .doReturn(easyRandom.objects(String.class, eachReturnSize).collect(Collectors.toList()))
                .when(someManager)
                .aListMethod(anyLong(), any());

        // 分批执行
        int size = 4;
        List<Integer> resultList = ExecuteUtil.partitionCall2List(mockDataList, size, (eachList) -> someManager.aListMethod(2L, eachList));

        //验证执行次数
        int invocations = 8;
        Mockito.verify(someManager, new Times(invocations)).aListMethod(anyLong(), any());

        // 正好几轮
        int turns;
        if (total % size == 0) {
            turns = total / size;
        } else {
            turns = total / size + 1;
        }
        Assert.assertEquals(turns * eachReturnSize, resultList.size());
    }


    @Test
    public void test_call_return_map_partition() {
        // mock  每次调用返回条数
        // 注意：
        // 如果仅调用doReturn一次，那么每次返回都是key相同的Map，
        // 如果需要不覆盖，则doReturn次数和 invocations 相同）
        int eachReturnSize = 3;
        PowerMockito
                .doReturn(mockMap(eachReturnSize))
                .doReturn(mockMap(eachReturnSize))
                .when(someManager).aMapMethod(anyLong(), any());

        // 每批
        int size = 16;
        Map<String, Integer> resultMap = ExecuteUtil.partitionCall2Map(mockDataList, size, (eachList) -> someManager.aMapMethod(2L, eachList));

        //验证执行次数
        int invocations = 2;
        Mockito.verify(someManager, new Times(invocations)).aMapMethod(anyLong(), any());

        // 正好几轮
        int turns;
        if (total % size == 0) {
            turns = total / size;
        } else {
            turns = total / size + 1;
        }
        Assert.assertEquals(turns * eachReturnSize, resultMap.size());
    }

    private Map<String, Integer> mockMap(int size) {
        Map<String, Integer> result = new HashMap<>(size);
        for (int i = 0; i < size; i++) {

// 极力保证key不重复
            result.put(easyRandom.nextObject(String.class) + RandomUtils.nextInt(), easyRandom.nextInt());
        }
        return result;
    }


}

注意：

1 判空

.filter(Objects::nonNull)

这里非常重要，避免又一次调用返回 null，而导致空指针异常。

2 实际使用时可以结合apollo配置，灵活设置每批执行的数量，如果超时随时调整

3 用到的类库

集合工具类： commons-collections4、guava （可以不用）

这里的list划分子list也可以使用stream的 skip ,limit特性自己去做，集合判空也可以不借助collectionutils.

构造数据：easy-random

单元测试框架： Junit4 、 powermockito、mockito

4 大家可以加一些更强大的功能，如允许设置每次调用的时间间隔、并行或并发调用等。

三、改进

以上面的List接口为例，将其改为异步版本：

    public static <T, V> List<V> partitionCall2ListAsync(List<T> dataList,
                                                         int size,
                                                         ExecutorService executorService,
                                                         Function<List<T>, List<V>> function) {

        if (CollectionUtils.isEmpty(dataList)) {
            return new ArrayList<>(0);
        }
        Preconditions.checkArgument(size > 0, "size must not be a minus");

        List<CompletableFuture<List<V>>> completableFutures = Lists.partition(dataList, size)
                .stream()
                .map(eachList -> {
                    if (executorService == null) {
                        return CompletableFuture.supplyAsync(() -> function.apply(eachList));
                    } else {
                        return CompletableFuture.supplyAsync(() -> function.apply(eachList), executorService);
                    }

                })
                .collect(Collectors.toList());


        CompletableFuture<Void> allFinished = CompletableFuture.allOf(completableFutures.toArray(new CompletableFuture[0]));
        try {
            allFinished.get();
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
        return completableFutures.stream()
                .map(CompletableFuture::join)
                .filter(CollectionUtils::isNotEmpty)
                .reduce(new ArrayList<V>(), ((list1, list2) -> {
                    List<V> resultList = new ArrayList<>();
                    if(CollectionUtils.isNotEmpty(list1)){
                       resultList.addAll(list1);
                       }

                    if(CollectionUtils.isNotEmpty(list2)){
                         resultList.addAll(list2);
                       }
                    return resultList;
                }));
    }

测试代码：

    
 // 测试数据
    private List<String> mockDataList;

    private int total = 300;

    private AtomicInteger atomicInteger;

    @Before
    public void init() {
        // 构造total条数据
        mockDataList = easyRandom.objects(String.class, total).collect(Collectors.toList());

    }



@Test
 public void test_call_return_list_partition_async() {

        ExecutorService executorService = Executors.newFixedThreadPool(10);

        atomicInteger = new AtomicInteger(0);
        Stopwatch stopwatch = Stopwatch.createStarted();
        // 分批执行
        int size = 2;
        List<Integer> resultList = ExecuteUtil.partitionCall2ListAsync(mockDataList, size, executorService, (eachList) -> someCall(2L, eachList));

        Stopwatch stop = stopwatch.stop();
        log.info("执行时间: {} 秒", stop.elapsed(TimeUnit.SECONDS));

        Assert.assertEquals(total, resultList.size());
        // 正好几轮
        int turns;
        if (total % size == 0) {
            turns = total / size;
        } else {
            turns = total / size + 1;
        }
        log.info("共调用了{}次", turns);
        Assert.assertEquals(turns, atomicInteger.get());
    
      // 顺序也一致
        for(int i =0; i< mockDataList.size();i++){
            Assert.assertEquals((Integer) mockDataList.get(i).length(), resultList.get(i));
        }
    }


  /**
     * 模拟一次调用
     */
    private List<Integer> someCall(Long id, List<String> strList) {

        log.info("当前-->{}，strList.size：{}", atomicInteger.incrementAndGet(), strList.size());
        try {
            TimeUnit.SECONDS.sleep(2L);
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
        return strList.stream()
                .map(String::length)
                .collect(Collectors.toList());
    }