Hi team, I have a pyflink application which reads ...
# troubleshooting
a
Hi team, I have a pyflink application which reads from Kafka and sinks the data to Kafka PRoducer and ElasticSearch. Below is my flink configutaion:
Copy code
flinkConfiguration:
    metrics.reporter.prom.class: org.apache.flink.metrics.prometheus.PrometheusReporter
    metrics.reporter.prom.factory.class: org.apache.flink.metrics.prometheus.PrometheusReporterFactory
    metrics.reporter.prom.port: "9100"
    metrics.reporters: prom
    taskmanager.network.detailed-metrics: "true"
    taskmanager.numberOfTaskSlots: "3"
The problem is I am not able to get taskmanager metrics in prometheus, as soon as the job starts runnings the taskmanager metrics disappear. This application is deployed on a kubernetes and I am using flink 1.18.0 Here is my code:
Copy code
def consumer():
    return FlinkKafkaConsumer(
        topics=config.TOPICS.split(","),
        deserialization_schema=SimpleStringSchema(),
        properties={
            "bootstrap.servers": config.KAFKA_BOOTSTRAP_SERVER,
            "security.protocol": config.KAFKA_SECURITY_PROTOCOL,
            "sasl.mechanism": config.KAFKA_SASL_MECHANISM,
            "group.id": "test",
            "enable.auto.commit": "false",
            "reconnect.backoff.ms": "1000",
            "reconnect.backoff.max.ms": "5000",
            "sasl.jaas.config": f'org.apache.kafka.common.security.plain.PlainLoginModule required username="{config.KAFKA_USERNAME}" password="{config.KAFKA_PASSWORD}";',
        },
    )

def producer():
    return FlinkKafkaProducer(
        topic=config.PRODUCER_TOPIC,
        serialization_schema=SimpleStringSchema(),
        producer_config={
            "bootstrap.servers": config.PRODUCER_KAFKA_BOOTSTRAP_SERVER,
            "security.protocol": config.KAFKA_SECURITY_PROTOCOL,
            "sasl.mechanism": config.KAFKA_SASL_MECHANISM,
            "group.id": "test",
            "sasl.jaas.config": f'org.apache.kafka.common.security.plain.PlainLoginModule required username="{config.PRODUCER_KAFKA_USERNAME}" password="{config.PRODUCER_KAFKA_PASSWORD}";',
            "auto.commit.enable": "false",
        },
        semantic=Semantic.AT_LEAST_ONCE,
    )

def stream_data():
    env = StreamExecutionEnvironment.get_execution_environment()
    env.add_jars("file:///opt/flink/usrlib/flink-sql-connector-kafka-1.15.0.jar")
    env.add_jars("file:///opt/flink/usrlib/kafka-clients-3.6.0.jar")
    env.add_jars("file:///opt/flink/usrlib/flink-sql-connector-elasticsearch7-3.0.1-1.17.jar") 
    kafka_consumer = consumer()
    kafka_producer = producer()
    env.set_parallelism(4)
    env.enable_checkpointing(config.CHECKPOINT_INTERVAL)
    env.get_checkpoint_config().set_checkpoint_timeout(120000)
    env.get_checkpoint_config().set_checkpointing_mode(CheckpointingMode.EXACTLY_ONCE)
    env.get_checkpoint_config().set_min_pause_between_checkpoints(1000)
    env.get_checkpoint_config().set_max_concurrent_checkpoints(1)

    kafka_consumer.set_start_from_group_offsets()
    kafka_consumer.set_commit_offsets_on_checkpoints(True)
    logging.info("Ingesting datastream from Kafka")
    watermark_strategy = WatermarkStrategy.no_watermarks()
    ds = env.add_source(kafka_consumer, "Consumer stream").rebalance()
    ds = ds.assign_timestamps_and_watermarks(watermark_strategy)
    ds = ds.flat_map(utils.process_logs).filter(lambda x: x is not None).key_by(extract_key)
    es7_sink = Elasticsearch7SinkBuilder() \
        .set_emitter(ElasticsearchEmitter.static_index(config.INDEX_NAME, 'eventId')) \
        .set_hosts([f'https://{config.OPENSEARCH_HOST}:9200']) \
        .set_bulk_flush_max_actions(250) \
        .set_connection_username(config.OPENSEARCH_USERNAME) \
        .set_connection_password(config.OPENSEARCH_PASSWORD) \
        .set_bulk_flush_backoff_strategy(FlushBackoffType.CONSTANT, 3, 50) \
        .build()
    
    ds.map(lambda x: x, Types.MAP(Types.STRING(), Types.STRING())).sink_to(es7_sink).name("Sink to Opensearch")
    ds.map(lambda x: json.dumps(x, cls=utils.DateTimeEncoder), Types.STRING()).add_sink(kafka_producer)
    env.execute("Audit loging")