Hi Team, i am running flink locally (kafka -> f...
# troubleshooting
s
Hi Team, i am running flink locally (kafka -> flink -> minio) but facing issues with s3
Caused by: org.apache.hadoop.fs.UnsupportedFileSystemException: No FileSystem for scheme "s3"
r
are you using Flink SQL? or your own java code?
s
KafkaFlink.java
Copy code
package com.kafka.flink;

import org.apache.flink.api.common.eventtime.WatermarkStrategy;
import org.apache.flink.api.common.serialization.SimpleStringSchema;
import org.apache.flink.connector.kafka.source.KafkaSource;
import org.apache.flink.connector.kafka.source.enumerator.initializer.OffsetsInitializer;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.table.data.RowData;
import org.apache.iceberg.DistributionMode;
import org.apache.iceberg.catalog.TableIdentifier;
import org.apache.iceberg.flink.CatalogLoader;
import org.apache.iceberg.flink.TableLoader;
import org.apache.iceberg.flink.sink.FlinkSink;
import org.apache.hadoop.conf.Configuration;

import java.util.HashMap;
import java.util.Map;

public class KafkaFlink {

    public static void main(String[] args) throws Exception {

        final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

        KafkaSource<String> kafkaSource = KafkaSource.<String>builder()
                .setBootstrapServers("localhost:9092")
                .setTopics("test-topic")
                .setGroupId("flink-consumer-group")
                .setStartingOffsets(OffsetsInitializer.earliest())
                .setValueOnlyDeserializer(new SimpleStringSchema())
                .build();

        DataStream<String> kafkaStream = env.fromSource(kafkaSource, WatermarkStrategy.noWatermarks(), "Kafka Source");

        DataStream<RowData> rowDataStream = kafkaStream.map(new KafkaJsonToRowDataMapper());

        TableIdentifier tableIdentifier = TableIdentifier.of("flink", "kafka_table");
        TableLoader tableLoader = TableLoader.fromCatalog(getCatalogLoader(), tableIdentifier);
        tableLoader.open();

        FlinkSink.forRowData(rowDataStream)
                .tableLoader(tableLoader)
                .toBranch("main")
                .distributionMode(DistributionMode.HASH)
                .upsert(false)
                .append();

        env.execute("Kafka to Iceberg Streaming Job");
    }

    public static CatalogLoader getCatalogLoader() {
        Map<String, String> props = new HashMap<>();
        props.put("uri", "http://localhost:19120/api/v1");
        props.put("ref", "main");
        props.put("warehouse", "s3a://warehouse/");
        props.put("s3.endpoint", "http://localhost:9000");
        props.put("s3.region", "us-east-1");
        props.put("s3.access-key", "admin");
        props.put("s3.secret-key", "password");
        props.put("s3.path-style-access", "true");
        props.put("fs.s3a.impl", "org.apache.hadoop.fs.s3a.S3AFileSystem");
        props.put("write.format.default", "parquet");

        // return CatalogLoader.custom("nessie", props, new Configuration(), "org.apache.iceberg.nessie.NessieCatalog");
        org.apache.hadoop.conf.Configuration hadoopConf = new org.apache.hadoop.conf.Configuration();

        return CatalogLoader.custom(
                "nessie",
                props,
                hadoopConf, "org.apache.iceberg.nessie.NessieCatalog");
    }
}
helper class
Copy code
package com.kafka.flink;

import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.table.data.GenericRowData;
import org.apache.flink.table.data.StringData;
import org.apache.flink.table.data.RowData;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;

public class KafkaJsonToRowDataMapper implements MapFunction<String, RowData> {
    private static final ObjectMapper objectMapper = new ObjectMapper();

    @Override
    public RowData map(String value) throws Exception {
        JsonNode node = objectMapper.readTree(value);
        String id = node.get("id").asText();
        String name = node.has("name") ? node.get("name").asText() : "";
        int clicks = node.has("clicks") ? node.get("clicks").asInt() : 0;
        String createdAt = node.has("created_at") ? node.get("created_at").asText() : "";

        GenericRowData rowData = new GenericRowData(4);
        rowData.setField(0, StringData.fromString(id));
        rowData.setField(1, StringData.fromString(name));
        rowData.setField(2, clicks);
        rowData.setField(3, StringData.fromString(createdAt));
        return rowData;
    }
}
flink-conf.yaml
Copy code
rest.address: localhost
rest.port: 8081
jobmanager.rpc.address: localhost


# TaskManager Memory Settings
taskmanager.memory.process.size: 2300m
taskmanager.memory.flink.size: 2048m
taskmanager.memory.managed.size: 512m
taskmanager.memory.task.heap.size: 1024m
taskmanager.memory.jvm-metaspace.size: 128m
taskmanager.memory.jvm-overhead.min: 120m
taskmanager.memory.jvm-overhead.max: 256m

# JobManager Memory Settings
jobmanager.memory.process.size: 1024m
jobmanager.memory.heap.size: 512m
jobmanager.memory.off-heap.size: 256m
jobmanager.memory.jvm-metaspace.size: 128m
jobmanager.memory.jvm-overhead.min: 128m
jobmanager.memory.jvm-overhead.max: 256m


s3.endpoint: http://127.0.0.1:9000
s3.path-style: true
s3.access-key: admin
s3.secret-key: password
s3.region: us-east-1
s3.path.style.access: "true"

fs.s3a.access.key: "admin"
fs.s3a.secret.key: "password"
fs.s3a.endpoint: "http://localhost:9000"
fs.s3a.path.style.access: "true"
fs.s3.impl: org.apache.hadoop.fs.s3a.S3AFileSystem
@rmoff thanks for replying, it is java code shared above ^
r
ah ok, sorry then - i only know SQL πŸ™‚ just in case, here are the two articles I wrote that might be useful still: https://www.decodable.co/blog/flink-sql-misconfiguration-misunderstanding-and-mishaps#writing-to-s3-from-flink https://www.decodable.co/blog/troubleshooting-flink-sql-s3-problems
πŸ‘€ 1
s
thanks @rmoff looks like io-impl property was missing from my side
Copy code
public static CatalogLoader getCatalogLoader() {
        MapString, String props = new HashMap<>();
        props.put("uri", "http://localhost:19120/api/v1");
        props.put("ref", "main");
        props.put("warehouse", "s3a://warehouse/");
        props.put("s3.endpoint", "http://localhost:9000");
        // props.put("s3.region", "us-east-1");
        props.put("s3.access-key", "admin");
        props.put("s3.secret-key", "password");
        props.put("s3.path-style-access", "true");
        props.put("io-impl", "org.apache.iceberg.aws.s3.S3FileIO");
        // props.put("s3.use-arn-region-enabled", "true");
        // props.put("fs.s3a.impl", "org.apache.hadoop.fs.s3a.S3AFileSystem");
        props.put("write.format.default", "parquet");

        // return CatalogLoader.custom("nessie", props, new Configuration(), "org.apache.iceberg.nessie.NessieCatalog");
        org.apache.hadoop.conf.Configuration hadoopConf = new org.apache.hadoop.conf.Configuration();

        return CatalogLoader.custom(
                "nessie",
                props,
                hadoopConf, "org.apache.iceberg.nessie.NessieCatalog");
    }
I am getting below error now even if i give aws creds
Copy code
export AWS_ACCESS_KEY_ID=admin
export AWS_SECRET_ACCESS_KEY=password
export AWS_REGION=us-east-1
export AWS_DEFAULT_REGION=us-east-1
Copy code
Job has been submitted with JobID 6f33f1b4ab1fa9fd1b9d90d9b12a3904

------------------------------------------------------------
 The program finished with the following exception:

	at java.base/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1594)
	at java.base/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:183)
Caused by: software.amazon.awssdk.core.exception.SdkClientException: Unable to load credentials from any of the providers in the chain AwsCredentialsProviderChain(credentialsProviders=[SystemPropertyCredentialsProvider(), EnvironmentVariableCredentialsProvider(), WebIdentityTokenCredentialsProvider(), ProfileCredentialsProvider(profileName=default, profileFile=ProfileFile(sections=[])), ContainerCredentialsProvider(), InstanceProfileCredentialsProvider()]) : [SystemPropertyCredentialsProvider(): Unable to load credentials from system settings. Access key must be specified either via environment variable (AWS_ACCESS_KEY_ID) or system property (aws.accessKeyId)., EnvironmentVariableCredentialsProvider(): Unable to load credentials from system settings. Access key must be specified either via environment variable (AWS_ACCESS_KEY_ID) or system property (aws.accessKeyId)., WebIdentityTokenCredentialsProvider(): Either the environment variable AWS_WEB_IDENTITY_TOKEN_FILE or the javaproperty aws.webIdentityTokenFile must be set., ProfileCredentialsProvider(profileName=default, profileFile=ProfileFile(sections=[])): Profile file contained no credentials for profile 'default': ProfileFile(sections=[]), ContainerCredentialsProvider(): Cannot fetch credentials from container - neither AWS_CONTAINER_CREDENTIALS_FULL_URI or AWS_CONTAINER_CREDENTIALS_RELATIVE_URI environment variables are set., InstanceProfileCredentialsProvider(): Failed to load credentials from IMDS.]
s
You have to include hdfs file system deps , its not flink error but the code is not able to understand the filesystem
d
Hi Shubham