DataWorks OpenAPI 示例（元数据模块）

2022-03-06 21:37:25

以下demo内容已经较老，大家使用API建议移步官方API文档：https://help.aliyun.com/document_detail/173915.html
有详细的demo代码和线上调试功能

DataWorks OpenAPI 元数据demo

Client初始化

    String region = "cn-shanghai";
    IClientProfile profile = DefaultProfile.getProfile(region, "xxx", "xxx");
    DefaultProfile.addEndpoint(region,"dataworks-public", String.format("dataworks.%s.aliyuncs.com", region));
    IAcsClient client = new DefaultAcsClient(profile);

GetMetaDBInfo

获取odps引擎详情

    GetMetaDBInfoRequest request = new GetMetaDBInfoRequest();
    //odps app guid，格式 odps.{projectName}
    request.setAppGuid("odps.project1");
    //类型为odps
    request.setDataSourceType("odps");
    GetMetaDBInfoResponse res = client.getAcsResponse(request);
    GetMetaDBInfoResponse.Data data = res.getData();

获取emr库详情

    GetMetaDBInfoRequest request = new GetMetaDBInfoRequest();
    request.setDataSourceType("emr");
    //emr 集群id
    request.setClusterId("C-C206DE8CACxxxxx");
    //emr db名称
    request.setDatabaseName("default");

    GetMetaDBInfoResponse res = client.getAcsResponse(request);
    GetMetaDBInfoResponse.Data data = res.getData();

ListMetaDB

获取emr 数据库列表

    ListMetaDBRequest request = new ListMetaDBRequest();
    //数据类型
    request.setDataSourceType("emr");
    //dataworks 项目id
    request.setProjectId(110L);
    request.setPageSize(1);
    request.setPageNum(10);

    ListMetaDBResponse res = client.getAcsResponse(request);
    //总条数
    res.getDatabaseInfo().getTotalCount();
    //数据
    res.getDatabaseInfo().getDbList();

GetMetaDBTableList

获取odps引擎关联表

    GetMetaDBTableListRequest request = new GetMetaDBTableListRequest();
    //app guid
    request.setAppGuid("odps.project1");
    request.setDataSourceType("odps");
    //第1页
    request.setPageNumber(1);
    //每页大小
    request.setPageSize(10);
    GetMetaDBTableListResponse res =  client.getAcsResponse(request);
    //数据总大小
    Long totalCount = res.getData().getTotalCount();
    //当前页数据列表
    List<GetMetaDBTableListResponse.Data.TableEntityListItem>  tableEntityListItems = res.getData().getTableEntityList();

获取emr 数据库下的表

    GetMetaDBTableListRequest request = new GetMetaDBTableListRequest();
    //数据类型
    request.setDataSourceType("emr");
    //emr集群id
    request.setClusterId("C-C206DE8CAC0xxxxx");
    //emr 数据库名称
    request.setDatabaseName("default");

    GetMetaDBTableListResponse res = client.getAcsResponse(request);
    //数据总大小
    Long totalCount = res.getData().getTotalCount();
    //当前页数据列表
    List<GetMetaDBTableListResponse.Data.TableEntityListItem>  tableEntityListItems = res.getData().getTableEntityList();

CheckMetaTable

检查odps表是否存在

    CheckMetaTableRequest request = new CheckMetaTableRequest();
    //odps table  guid，格式odps.{projectName}.{tableName}
    request.setTableGuid("odps.project1.t1");
    //资源类型
    request.setDataSourceType("odps");
    CheckMetaTableResponse res = client.getAcsResponse(request);
    //表是否存在
    Boolean exists = res.getData();

检查emr表是否存在

    CheckMetaTableRequest request = new CheckMetaTableRequest();
    //数据类型
    request.setDataSourceType("emr");
    //emr集群id
    request.setClusterId("C-C206DE8CXXXXXXX");
    //emr 数据库名称
    request.setDatabaseName("default");
    //表名称
    request.setTableName("table1");

    CheckMetaTableResponse res = client.getAcsResponse(request);

CheckMetaPartition

检查odps分区是否存在

    CheckMetaPartitionRequest request = new CheckMetaPartitionRequest();
    //odps table  guid，格式odps.{projectName}.{tableName}
    request.setTableGuid("odps.project1.t1");
    //资源类型
    request.setDataSourceType("odps");
    //分区
    request.setPartition("dt_time=20200419");

    CheckMetaPartitionResponse res = client.getAcsResponse(request);
    //分区是否存在
    Boolean exists = res.getData();

检查emr分区是否存在

    CheckMetaPartitionRequest request = new CheckMetaPartitionRequest();
    //数据类型
    request.setDataSourceType("emr");
    //emr集群id
    request.setClusterId("C-C206DE8CXXXXXXX");
    //emr 数据库名称
    request.setDatabaseName("default");
    //表名称
    request.setTableName("table1");

    request.setPartition("ds=202001");

    CheckMetaPartitionResponse res = client.getAcsResponse(request);
    //分区是否存在
    Boolean exists = res.getData();

SearchMetaTables

搜索odps表

    SearchMetaTablesRequest request = new SearchMetaTablesRequest();
    //odps app guid，格式 odps.{projectName}
    request.setAppGuid("odps.p1");
    request.setDataSourceType("odps");
    //表类型，0:table, 1:view  默认全部
    request.setEntityType(0);
    //关键字
    request.setKeyword("test");
    //分页
    request.setPageNumber(1);
    request.setPageSize(10);

    SearchMetaTablesResponse res = client.getAcsResponse(request);
    //数据总条数
    Long totalCount = res.getData().getTotalCount();
    //当前页表列表
    res.getData().getDataEntityList();

搜索emr表

    SearchMetaTablesRequest request = new SearchMetaTablesRequest();
    //数据类型
    request.setDataSourceType("emr");
    //emr集群id
    request.setClusterId("C-C206DE8CXXXXXXX");
    request.setKeyword("test");
    request.setPageNumber(1);
    request.setPageSize(10);

    SearchMetaTablesResponse res = client.getAcsResponse(request);
    //数据总条数
    Long totalCount = res.getData().getTotalCount();
    //当前页表列表
    res.getData().getDataEntityList();

GetMetaTableBasicInfo

获取odps表基本信息

    GetMetaTableBasicInfoRequest request = new GetMetaTableBasicInfoRequest();
    //odps table  guid，格式odps.{projectName}.{tableName}
    request.setTableGuid("odps.project1.t1");
    //资源类型
    request.setDataSourceType("odps");

    GetMetaTableBasicInfoResponse res = client.getAcsResponse(request);
    //表数据
    GetMetaTableBasicInfoResponse.Data info = res.getData();

获取emr表基础信息

    GetMetaTableBasicInfoRequest request = new GetMetaTableBasicInfoRequest();
    //数据类型
    request.setDataSourceType("emr");
    //emr集群id
    request.setClusterId("C-C206DE8CXXXXXXX");
    //emr 数据库名称
    request.setDatabaseName("default");
    //表名称
    request.setTableName("table1");

    GetMetaTableBasicInfoResponse res = client.getAcsResponse(request);
    //表数据
    GetMetaTableBasicInfoResponse.Data info = res.getData();

GetMetaTableColumn

获取odps column列表

    GetMetaTableColumnRequest request = new GetMetaTableColumnRequest();
    //odps table  guid，格式odps.{projectName}.{tableName}
    request.setTableGuid("odps.project1.t1");
    //资源类型
    request.setDataSourceType("odps");
    //分页
    request.setPageNum(1);
    request.setPageSize(10);
    GetMetaTableColumnResponse res = client.getAcsResponse(request);
    //数据总条数
    Long totalCount = res.getData().getTotalCount();
    //当前页数据列表
    List<GetMetaTableColumnResponse.Data.ColumnListItem> columnList = res.getData().getColumnList();

获取emr表column信息

    GetMetaTableColumnRequest request = new GetMetaTableColumnRequest();
    //数据类型
    request.setDataSourceType("emr");
    //emr集群id
    request.setClusterId("C-C206DE8CXXXXXXX");
    //emr 数据库名称
    request.setDatabaseName("default");
    //表名称
    request.setTableName("table1");
    request.setPageNum(1);
    request.setPageSize(10);
    GetMetaTableColumnResponse res = client.getAcsResponse(request);
    //数据总条数
    Long totalCount = res.getData().getTotalCount();
    //当前页数据列表
    List<GetMetaTableColumnResponse.Data.ColumnListItem> columnList = res.getData().getColumnList();

GetMetaTablePartition

获取odps表分区

    GetMetaTablePartitionRequest request = new GetMetaTablePartitionRequest();
    //odps table  guid，格式odps.{projectName}.{tableName}
    request.setTableGuid("odps.project1.t1");
    //资源类型
    request.setDataSourceType("odps");
    //分页
    request.setPageNumber(1);
    request.setPageSize(10);
    GetMetaTablePartitionResponse res = client.getAcsResponse(request);
    //数据总条数
    res.getData().getTotalCount();
    //分区列表
    res.getData().getDataEntityList();

获取emr表分区

    GetMetaTablePartitionRequest request = new GetMetaTablePartitionRequest();
    //数据类型
    request.setDataSourceType("emr");
    //emr集群id
    request.setClusterId("C-C206DE8CXXXXXXX");
    //emr 数据库名称
    request.setDatabaseName("default");
    //表名称
    request.setTableName("table1");
    request.setPageNumber(1);
    request.setPageSize(10);

    GetMetaTablePartitionResponse res = client.getAcsResponse(request);
    //数据总条数
    res.getData().getTotalCount();
    //分区列表
    res.getData().getDataEntityList();

GetMetaTableOutput

获取odps表输出

    GetMetaTableOutputRequest request = new GetMetaTableOutputRequest();
    //odps table  guid，格式odps.{projectName}.{tableName}
    request.setTableGuid("odps.project1.t1");
    //output开始时间
    request.setStartDate("2020-05-27");
    //output结束时间
    request.setEndDate("2020-06-12");
    request.setPageNumber(1);
    request.setPageSize(10);
    GetMetaTableOutputResponse res = client.getAcsResponse(request);
    //数据总条数
    res.getData().getTotalCount();
    //输出列表
    res.getData().getDataEntityList();

GetMetaTableChangeLog

获取odps表变更信息

    GetMetaTableChangeLogRequest request = new GetMetaTableChangeLogRequest();
    //odps table  guid，格式odps.{projectName}.{tableName}
    request.setTableGuid("odps.project1.t1");
    //开始时间，格式yyyy-MM-dd HH:mm:ss
    request.setStartDate("2020-01-01 01:01:00");
    //格式yyyy-MM-dd HH:mm:ss
    request.setEndDate("2020-01-02 01:01:00");
    /**
     * 类型：
     * CREATE_TABLE
     * ALTER_TABLE
     * DROP_TABLE
     * ADD_PARTITION
     * DROP_PARTITION
     * ALTER_OWNER_ID
     * ALTER_LIFE_CYCLE_TIME
     */
    request.setChangeType("CREATE_TABLE");
    //对象 类型 TABLE，PARTITION
    request.setObjectType("TABLE");
    //分页
    request.setPageNumber(1);
    request.setPageSize(10);
    GetMetaTableChangeLogResponse res = client.getAcsResponse(request);
    //数据总条数
    res.getData().getTotalCount();
    //数据
    res.getData().getDataEntityList();

GetMetaTableIntroWiki

获取wiki

    GetMetaTableIntroWikiRequest request = new GetMetaTableIntroWikiRequest();
    //odps table  guid，格式odps.{projectName}.{tableName}
    request.setTableGuid("odps.project1.t1");
    //wiki版本
    request.setWikiVersion(1L);
    GetMetaTableIntroWikiResponse res = client.getAcsResponse(request);
    //数据
    res.getData();

GetMetaTableLineage

获取odps表血缘

    GetMetaTableLineageRequest request = new GetMetaTableLineageRequest();
    //odps table  guid，格式odps.{projectName}.{tableName}
    request.setTableGuid("odps.project1.t1");
    request.setDataSourceType("odps");
    //方向，up：上游 down：下游
    request.setDirection("up");
    //每页大小
    request.setPageSize(10);
    //下一页的起始key，null的时候从最开始查询
    String nextPrimaryKey = null;
    //是否有下一页
    Boolean hasNext = false;

    List<GetMetaTableLineageResponse.Data.DataEntityListItem> dataEntityListItems = new ArrayList<>();

    do {
        //分页起始key，返回数据中获取
        request.setNextPrimaryKey(nextPrimaryKey);

        GetMetaTableLineageResponse res = client.getAcsResponse(request);
        List<GetMetaTableLineageResponse.Data.DataEntityListItem> items = res.getData().getDataEntityList();
        dataEntityListItems.addAll(items);
        //是否有下一页
        hasNext = res.getData().getHasNext();
        //下一页起始key
        nextPrimaryKey = res.getData().getNextPrimaryKey();

    } while(hasNext);

获取emr表血缘

    GetMetaTableLineageRequest request = new GetMetaTableLineageRequest();
    //数据类型
    request.setDataSourceType("emr");
    //emr集群id
    request.setClusterId("C-C206DE8CXXXXXXX");
    //emr 数据库名称
    request.setDatabaseName("default");
    //表名称
    request.setTableName("table1");
    //下一次请求开始id，参考 GetMetaTableLineageResponse.getData().getNextPrimaryKey()
    request.setDirection("up");
    request.setPageSize(10);
    //下一页的起始key，null的时候从最开始查询
    String nextPrimaryKey = null;
    //是否有下一页
    Boolean hasNext = false;

    List<GetMetaTableLineageResponse.Data.DataEntityListItem> dataEntityListItems = new ArrayList<>();

    do {
        //分页起始key，返回数据中获取
        request.setNextPrimaryKey(nextPrimaryKey);

        GetMetaTableLineageResponse res = client.getAcsResponse(request);
        List<GetMetaTableLineageResponse.Data.DataEntityListItem> items = res.getData().getDataEntityList();
        dataEntityListItems.addAll(items);
        //是否有下一页
        hasNext = res.getData().getHasNext();
        //下一页起始key
        nextPrimaryKey = res.getData().getNextPrimaryKey();

    } while(hasNext);

GetMetaColumnLineage

获取odps字段血缘

    GetMetaColumnLineageRequest request = new GetMetaColumnLineageRequest();
    //column guid,格式 odps.{projectName}.{tableName}.{columnName}
    request.setColumnGuid("odps.project1.t1");
    request.setDataSourceType("odps");
    //方向，up：上游 down：下游
    request.setDirection("down");
    //分页
    request.setPageNum(1);
    request.setPageSize(10);
    GetMetaColumnLineageResponse res = client.getAcsResponse(request);
    //数据总条数
    res.getData().getTotalCount();
    //数据
    res.getData().getDataEntityList();

获取emr字段血缘

    GetMetaColumnLineageRequest request = new GetMetaColumnLineageRequest();
    //数据类型
    request.setDataSourceType("emr");
    //emr集群id
    request.setClusterId("C-C206DE8CXXXXXXX");
    //emr 数据库名称
    request.setDatabaseName("default");
    //表名称
    request.setTableName("table1");
    request.setDirection("up");
    request.setColumnName("eid");
    request.setPageNum(1);
    request.setPageSize(10);
    GetMetaColumnLineageResponse res = client.getAcsResponse(request);
    //数据总条数
    res.getData().getTotalCount();
    //数据
    res.getData().getDataEntityList();

CreateMetaCategory

创建类目

    CreateMetaCategoryRequest request = new CreateMetaCategoryRequest();
    //类目名称
    request.setName("test1");
    //注释
    request.setComment("test");
    //父类目id，0表示没有父类目
    request.setParentId(0L);

    CreateMetaCategoryResponse res = client.getAcsResponse(request);
    //生成的类目id
    res.getData().getCategoryId()

UpdateMetaCategory

更新类目

   UpdateMetaCategoryRequest request = new UpdateMetaCategoryRequest();
    request.setCategoryId(53664L);
    request.setName("t93233");
    request.setComment("the com");
    UpdateMetaCategoryResponse res = client.getAcsResponse(request);
    //是否成功
    res.getData();

DeleteMetaCategory

删除类目

    DeleteMetaCategoryRequest request = new DeleteMetaCategoryRequest();
    request.setCategoryId(53664L);
    DeleteMetaCategoryResponse res = client.getAcsResponse(request);
    //是否成功
    res.getData();

GetMetaCategory

获取子类目列表

    GetMetaCategoryRequest request = new GetMetaCategoryRequest();
    //父类目id
    request.setParentCategoryId(0L);
    request.setPageNum(1);
    request.setPageSize(10);
    GetMetaCategoryResponse res = client.getAcsResponse(request);
    //总数据
    res.getData().getTotalCount();
    //数据列表
    res.getData().getDataEntityList();

GetMetaTableListByCategory

获取类目关联表

   GetMetaTableListByCategoryRequest request = new GetMetaTableListByCategoryRequest();
    request.setCategoryId(53664L);
    request.setPageNumber(1);
    request.setPageSize(10);
    GetMetaTableListByCategoryResponse res = client.getAcsResponse(request);

AddToMetaCategory

增加类目、表关联

    AddToMetaCategoryRequest request = new AddToMetaCategoryRequest();
    request.setCategoryId(53664L);
    //odps table  guid，格式odps.{projectName}.{tableName}
    request.setTableGuid("odps.project1.t1");

    AddToMetaCategoryResponse res = client.getAcsResponse(request);

DeleteFromMetaCategory

删除类目、表关联

    DeleteFromMetaCategoryRequest request = new DeleteFromMetaCategoryRequest();
    request.setCategoryId(53642L);
    //odps table  guid，格式odps.{projectName}.{tableName}
    request.setTableGuid("odps.project1.t1");
    DeleteFromMetaCategoryResponse res = client.getAcsResponse(request);

码农公寓

DataWorks OpenAPI 元数据demo

Client初始化

GetMetaDBInfo

ListMetaDB

GetMetaDBTableList

CheckMetaTable

CheckMetaPartition

SearchMetaTables

GetMetaTableBasicInfo

GetMetaTableColumn

GetMetaTablePartition

GetMetaTableOutput

GetMetaTableChangeLog

GetMetaTableIntroWiki

GetMetaTableLineage

GetMetaColumnLineage

CreateMetaCategory

UpdateMetaCategory

DeleteMetaCategory

GetMetaCategory

GetMetaTableListByCategory

AddToMetaCategory

DeleteFromMetaCategory

相关文章