Shubham Tomar
03/03/2025, 10:20 AMCaused by: org.apache.hadoop.fs.UnsupportedFileSystemException: No FileSystem for scheme "s3"rmoff
03/03/2025, 10:21 AMShubham Tomar
03/03/2025, 10:22 AMpackage com.kafka.flink;
import org.apache.flink.api.common.eventtime.WatermarkStrategy;
import org.apache.flink.api.common.serialization.SimpleStringSchema;
import org.apache.flink.connector.kafka.source.KafkaSource;
import org.apache.flink.connector.kafka.source.enumerator.initializer.OffsetsInitializer;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.table.data.RowData;
import org.apache.iceberg.DistributionMode;
import org.apache.iceberg.catalog.TableIdentifier;
import org.apache.iceberg.flink.CatalogLoader;
import org.apache.iceberg.flink.TableLoader;
import org.apache.iceberg.flink.sink.FlinkSink;
import org.apache.hadoop.conf.Configuration;
import java.util.HashMap;
import java.util.Map;
public class KafkaFlink {
public static void main(String[] args) throws Exception {
final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
KafkaSource<String> kafkaSource = KafkaSource.<String>builder()
.setBootstrapServers("localhost:9092")
.setTopics("test-topic")
.setGroupId("flink-consumer-group")
.setStartingOffsets(OffsetsInitializer.earliest())
.setValueOnlyDeserializer(new SimpleStringSchema())
.build();
DataStream<String> kafkaStream = env.fromSource(kafkaSource, WatermarkStrategy.noWatermarks(), "Kafka Source");
DataStream<RowData> rowDataStream = kafkaStream.map(new KafkaJsonToRowDataMapper());
TableIdentifier tableIdentifier = TableIdentifier.of("flink", "kafka_table");
TableLoader tableLoader = TableLoader.fromCatalog(getCatalogLoader(), tableIdentifier);
tableLoader.open();
FlinkSink.forRowData(rowDataStream)
.tableLoader(tableLoader)
.toBranch("main")
.distributionMode(DistributionMode.HASH)
.upsert(false)
.append();
env.execute("Kafka to Iceberg Streaming Job");
}
public static CatalogLoader getCatalogLoader() {
Map<String, String> props = new HashMap<>();
props.put("uri", "http://localhost:19120/api/v1");
props.put("ref", "main");
props.put("warehouse", "s3a://warehouse/");
props.put("s3.endpoint", "http://localhost:9000");
props.put("s3.region", "us-east-1");
props.put("s3.access-key", "admin");
props.put("s3.secret-key", "password");
props.put("s3.path-style-access", "true");
props.put("fs.s3a.impl", "org.apache.hadoop.fs.s3a.S3AFileSystem");
props.put("write.format.default", "parquet");
// return CatalogLoader.custom("nessie", props, new Configuration(), "org.apache.iceberg.nessie.NessieCatalog");
org.apache.hadoop.conf.Configuration hadoopConf = new org.apache.hadoop.conf.Configuration();
return CatalogLoader.custom(
"nessie",
props,
hadoopConf, "org.apache.iceberg.nessie.NessieCatalog");
}
}
helper class
package com.kafka.flink;
import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.table.data.GenericRowData;
import org.apache.flink.table.data.StringData;
import org.apache.flink.table.data.RowData;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
public class KafkaJsonToRowDataMapper implements MapFunction<String, RowData> {
private static final ObjectMapper objectMapper = new ObjectMapper();
@Override
public RowData map(String value) throws Exception {
JsonNode node = objectMapper.readTree(value);
String id = node.get("id").asText();
String name = node.has("name") ? node.get("name").asText() : "";
int clicks = node.has("clicks") ? node.get("clicks").asInt() : 0;
String createdAt = node.has("created_at") ? node.get("created_at").asText() : "";
GenericRowData rowData = new GenericRowData(4);
rowData.setField(0, StringData.fromString(id));
rowData.setField(1, StringData.fromString(name));
rowData.setField(2, clicks);
rowData.setField(3, StringData.fromString(createdAt));
return rowData;
}
}
flink-conf.yaml
rest.address: localhost
rest.port: 8081
jobmanager.rpc.address: localhost
# TaskManager Memory Settings
taskmanager.memory.process.size: 2300m
taskmanager.memory.flink.size: 2048m
taskmanager.memory.managed.size: 512m
taskmanager.memory.task.heap.size: 1024m
taskmanager.memory.jvm-metaspace.size: 128m
taskmanager.memory.jvm-overhead.min: 120m
taskmanager.memory.jvm-overhead.max: 256m
# JobManager Memory Settings
jobmanager.memory.process.size: 1024m
jobmanager.memory.heap.size: 512m
jobmanager.memory.off-heap.size: 256m
jobmanager.memory.jvm-metaspace.size: 128m
jobmanager.memory.jvm-overhead.min: 128m
jobmanager.memory.jvm-overhead.max: 256m
s3.endpoint: http://127.0.0.1:9000
s3.path-style: true
s3.access-key: admin
s3.secret-key: password
s3.region: us-east-1
s3.path.style.access: "true"
fs.s3a.access.key: "admin"
fs.s3a.secret.key: "password"
fs.s3a.endpoint: "http://localhost:9000"
fs.s3a.path.style.access: "true"
fs.s3.impl: org.apache.hadoop.fs.s3a.S3AFileSystemShubham Tomar
03/03/2025, 10:22 AMrmoff
03/03/2025, 10:23 AMShubham Tomar
03/03/2025, 10:40 AMpublic static CatalogLoader getCatalogLoader() {
MapString, String props = new HashMap<>();
props.put("uri", "http://localhost:19120/api/v1");
props.put("ref", "main");
props.put("warehouse", "s3a://warehouse/");
props.put("s3.endpoint", "http://localhost:9000");
// props.put("s3.region", "us-east-1");
props.put("s3.access-key", "admin");
props.put("s3.secret-key", "password");
props.put("s3.path-style-access", "true");
props.put("io-impl", "org.apache.iceberg.aws.s3.S3FileIO");
// props.put("s3.use-arn-region-enabled", "true");
// props.put("fs.s3a.impl", "org.apache.hadoop.fs.s3a.S3AFileSystem");
props.put("write.format.default", "parquet");
// return CatalogLoader.custom("nessie", props, new Configuration(), "org.apache.iceberg.nessie.NessieCatalog");
org.apache.hadoop.conf.Configuration hadoopConf = new org.apache.hadoop.conf.Configuration();
return CatalogLoader.custom(
"nessie",
props,
hadoopConf, "org.apache.iceberg.nessie.NessieCatalog");
}
I am getting below error now even if i give aws creds
export AWS_ACCESS_KEY_ID=admin
export AWS_SECRET_ACCESS_KEY=password
export AWS_REGION=us-east-1
export AWS_DEFAULT_REGION=us-east-1
Job has been submitted with JobID 6f33f1b4ab1fa9fd1b9d90d9b12a3904
------------------------------------------------------------
The program finished with the following exception:
at java.base/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1594)
at java.base/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:183)
Caused by: software.amazon.awssdk.core.exception.SdkClientException: Unable to load credentials from any of the providers in the chain AwsCredentialsProviderChain(credentialsProviders=[SystemPropertyCredentialsProvider(), EnvironmentVariableCredentialsProvider(), WebIdentityTokenCredentialsProvider(), ProfileCredentialsProvider(profileName=default, profileFile=ProfileFile(sections=[])), ContainerCredentialsProvider(), InstanceProfileCredentialsProvider()]) : [SystemPropertyCredentialsProvider(): Unable to load credentials from system settings. Access key must be specified either via environment variable (AWS_ACCESS_KEY_ID) or system property (aws.accessKeyId)., EnvironmentVariableCredentialsProvider(): Unable to load credentials from system settings. Access key must be specified either via environment variable (AWS_ACCESS_KEY_ID) or system property (aws.accessKeyId)., WebIdentityTokenCredentialsProvider(): Either the environment variable AWS_WEB_IDENTITY_TOKEN_FILE or the javaproperty aws.webIdentityTokenFile must be set., ProfileCredentialsProvider(profileName=default, profileFile=ProfileFile(sections=[])): Profile file contained no credentials for profile 'default': ProfileFile(sections=[]), ContainerCredentialsProvider(): Cannot fetch credentials from container - neither AWS_CONTAINER_CREDENTIALS_FULL_URI or AWS_CONTAINER_CREDENTIALS_RELATIVE_URI environment variables are set., InstanceProfileCredentialsProvider(): Failed to load credentials from IMDS.]shivank chaturvedi
03/04/2025, 11:57 AMdeep pandey
03/10/2025, 10:39 AM