Yang osmondy
06/27/2023, 8:46 AMYang osmondy
06/27/2023, 9:06 AMsql-client.sh -i /data/flink/init.sql
create table:
-- create mongo table
CREATE TABLE flink_database.md_xxxbaseinfo (
`_id` String,
`str` String,
`time` TIMESTAMP_LTZ,
`arr1` array<ROW<`inv` String , `detail` ROW<`subSum` string , `aubSum` string , `subDetails` array<Map<String ,String >> , `aubDetails` array<Map<String ,String >> > , `invType_CN` String >> ,
`int` INT
) WITH (
'connector' = 'mongodb',
'uri' = '<mongodb://root:mingyang100@xxx:27017>,xxx:27017,xxx:27017/admin?replicaSet=rs0&connectTimeoutMS=10000&authSource=admin',
'database' = 'test',
'collection' = 'xxxBaseInfo'
);
-- create hive table
SET table.sql-dialect=hive;
CREATE external TABLE ods.ods_md_xxx_xxxBaseInfo(
`_id` string,
`str` string,
`time` TIMESTAMP,
`arr1` array<STRUCT<`inv`:String, `detail`:STRUCT<`subSum`:String, `aubSum`:String, `subDetails`: array<Map<String,String>>, `aubDetails`: array<Map<String,String>>>, `invType_CN`: String >>,
`int` int
)
ROW FORMAT SERDE
'org.apache.hadoop.hive.ql.io.orc.OrcSerde'
WITH SERDEPROPERTIES (
'collection.delim'=',',
'field.delim'='\t',
'serialization.format'='\t')
STORED AS ORC
LOCATION
'<hdfs://xxx:8020/warehouse/tablespace/external/hive/ods.db/mongo/xxx/xxxBaseInfo>';
-- sync data
SET table.sql-dialect=default;
SET 'table.local-time-zone' = 'UTC';
INSERT INTO ods.ods_md_xxx_xxxBaseInfo
SELECT *
FROM flink_database.md_xxxbaseinfo;Alexander Fedulov
06/27/2023, 9:07 AMAlexander Fedulov
06/27/2023, 9:09 AMYang osmondy
06/27/2023, 9:49 AMmaster/slave . I have read the relevant official documents, 'scan.partition.strategy'='default' ,
But i still don't know how to improve job parallelism?
My Flink Job and My machine resource usage:Yang osmondy
06/28/2023, 12:59 AMMartijn Visser
06/28/2023, 12:38 PMYang osmondy
06/28/2023, 12:42 PM