基于tunnel读取odps数据并转成实体

一、引入依赖

• odps

<dependency>
            <groupId>com.aliyun.odps</groupId>
            <artifactId>odps-sdk-core</artifactId>
            <version>0.38.3</version>
</dependency>

• hutool

<dependency>
            <groupId>cn.hutool</groupId>
            <artifactId>hutool-all</artifactId>
            <version>5.6.5</version>
</dependency>

• guava

<dependency>
            <groupId>com.google.guava</groupId>
            <artifactId>guava</artifactId>
            <version>23.0</version>
</dependency>

二、工具类

package com.demo.util;

import cn.hutool.core.util.StrUtil;
import com.aliyun.odps.Column;
import com.aliyun.odps.Odps;
import com.aliyun.odps.OdpsType;
import com.aliyun.odps.PartitionSpec;
import com.aliyun.odps.account.Account;
import com.aliyun.odps.account.AliyunAccount;
import com.aliyun.odps.data.Record;
import com.aliyun.odps.data.RecordReader;
import com.aliyun.odps.tunnel.TableTunnel;
import com.google.common.collect.Lists;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.springframework.util.Base64Utils;

import java.lang.reflect.Constructor;
import java.lang.reflect.Field;
import java.nio.charset.StandardCharsets;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;

/**
 * @author: mt
 * @description: 基于tunnel读取odps
 * @create: 9:27 2022/8/12
 */
@Slf4j
@Component
public class OdpsTunnelUtil<T> {

    @Value("${odps.accessId}")
    private String accessId;
    @Value("${odps.accessSecret}")
    private String accessSecret;
    @Value("${odps.endPint}")
    private String odpsEndPint;
    @Value("${odps.tunnel.endpoint}")
    private String tunnelEndPoint;

    private static final Map<String, Odps> odpsMap = new HashMap<>();
    private static final Map<String, Column> columnMap = new HashMap<>();

    public List<T> tunnel(String project, String table, String partition, long start, long limit, Class<T> clazz) {
        List<T> result = Lists.newArrayList();

        TableTunnel tunnel = new TableTunnel(getOdps(project));
        tunnel.setEndpoint(tunnelEndPoint);
        RecordReader recordReader = null;
        try {
            TableTunnel.DownloadSession downloadSession;
            if (StrUtil.isEmpty(partition)) {
                downloadSession = tunnel.createDownloadSession(project, table);
            } else {
                PartitionSpec partitionSpec = new PartitionSpec(partition);
                downloadSession = tunnel.createDownloadSession(project, table, partitionSpec);
            }
            if (downloadSession.getRecordCount() <= start) {
                return result;
            }
            recordReader = downloadSession.openRecordReader(start, limit);
            Record record;
            while ((record = recordReader.read()) != null) {
                Constructor<T> constructor = clazz.getConstructor();
                T entity = constructor.newInstance();
                Field[] declaredFields = clazz.getDeclaredFields();
                for (Field field : declaredFields) {
                    field.setAccessible(true);
                    Object value = getTransformValue(project, table, record, field);
                    if (!Objects.isNull(value)) {
                        field.set(entity, value);
                    }
                }
                result.add(entity);
            }
            return result;
        } catch (Exception e) {
            log.error("[OdpsTunnelUtil-tunnel] error,project:{},table:{},start:{},limit:{},partition:{}",
                    project, table, start, limit, partition);
            throw new RuntimeException(e);
        } finally {
            if (!Objects.isNull(recordReader)) {
                try {
                    recordReader.close();
                } catch (Exception e) {
                    log.error("[OdpsTunnelUtil-tunnel] close readReader fail", e);
                }
            }
        }
    }


    private Odps getOdps(String project) {
        if (!Objects.isNull(odpsMap.get(project))) {
            return odpsMap.get(project);
        }

        accessId = new String(Base64Utils.decodeFromString(accessId), StandardCharsets.UTF_8);
        accessSecret = new String(Base64Utils.decodeFromString(accessSecret), StandardCharsets.UTF_8);

        Account account = new AliyunAccount(accessId, accessSecret);
        Odps odps = new Odps(account);
        odps.setEndpoint(odpsEndPint);
        odps.setDefaultProject(project);
        odpsMap.put(project, odps);
        return odps;
    }

    private Object getTransformValue(String project, String table, Record record, Field field) {
        String dbField = StrUtil.toUnderlineCase(field.getName());
        String key = project + table + dbField;
        Column column = getRightColumn(project, table, record, key);
        if (column == null) {
            return null;
        }

        OdpsType odpsType = column.getTypeInfo().getOdpsType();
        if (odpsType == OdpsType.STRING) {
            return record.getString(dbField);
        } else if (odpsType == OdpsType.BOOLEAN) {
            return record.getBoolean(dbField);
        } else if (odpsType == OdpsType.DOUBLE || odpsType == OdpsType.FLOAT) {
            return record.getDouble(dbField);
        } else if (odpsType == OdpsType.DECIMAL) {
            return record.getDecimal(dbField);
        } else if (odpsType == OdpsType.INT || odpsType == OdpsType.TINYINT) {
            return record.getBigint(dbField).intValue();
        } else if (odpsType == OdpsType.BIGINT) {
            if ("Integer".equalsIgnoreCase(field.getType().getSimpleName())) {
                return record.getBigint(dbField).intValue();
            }
            return record.getBigint(dbField);
        } else if (odpsType == OdpsType.DATE || odpsType == OdpsType.DATETIME) {
            return record.getDatetime(dbField);
        }
        return null;

    }

    private Column getRightColumn(String project, String table, Record record, String key) {
        if (!columnMap.containsKey(key)) {
            Column[] columns = record.getColumns();
            for (Column column : columns) {
                columnMap.put(project + table + column.getName(), column);
            }
        }

        return columnMap.get(key);
    }
}