Hell guys I just implemented Basic Auth to my Pino...
# troubleshooting
y
Hell guys I just implemented Basic Auth to my Pinot but while inserting data in tables using apache flinks stream with this code public static void insertIntoPinot(DataStream<Row> sinkStream, String sinkTableName, String controllerURL) throws Exception { HttpClient httpClient = HttpClient.getInstance(); ControllerRequestClient client = new ControllerRequestClient( ControllerRequestURLBuilder.baseUrl(controllerURL), httpClient); Schema schema = PinotConnectionUtils.getSchema(client, sinkTableName); TableConfig tableConfig = PinotConnectionUtils.getTableConfig(client, sinkTableName, "OFFLINE"); Logging.log(jobName, "PROCESS", "Job Inserting Data into Pinot " + controllerURL); sinkStream.addSink( new PinotSinkFunction<>( new FlinkRowGenericRowConverter(TYPE_INFO), tableConfig, schema)) .name("PinotSink_" + sinkTableName) .setParallelism(PARALLELISM); Logging.log(jobName, "PROCESS", "Job Inserted Data into Pinot " + controllerURL); } but this connectors/drivers don't have an option for basic auth and passing of headers how could I ingest my data now, I also tried basic HTTP client request to ingest data but I was running into issues with batch ingest config import java.net.URI; import java.net.http.HttpClient; import java.net.http.HttpRequest; import java.net.http.HttpResponse; import java.util.Base64; import java.util.ArrayList; import java.util.List; import java.util.Map; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.pinot.spi.config.table.TableConfig; import org.apache.pinot.spi.config.table.ingestion.BatchIngestionConfig; import org.apache.pinot.spi.config.table.ingestion.IngestionConfig; import org.apache.pinot.spi.data.Schema; import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.types.Row; public static void insertIntoPinot(DataStream<Row> sinkStream, String sinkTableName, String controllerURL, String username, String password) throws Exception { // Encode credentials for basic auth String authString = username + ":" + password; String encodedAuth = Base64.getEncoder().encodeToString(authString.getBytes()); HttpClient httpClient = HttpClient.newHttpClient(); ObjectMapper mapper = new ObjectMapper(); // 1) Fetch schema JSON from Pinot Controller REST API String schemaUrl = controllerURL + "/schemas/" + sinkTableName; HttpRequest schemaRequest = HttpRequest.newBuilder() .uri(URI.create(schemaUrl)) .header("Authorization", "Basic " + encodedAuth) .GET() .build(); HttpResponse<String> schemaResponse = httpClient.send(schemaRequest, HttpResponse.BodyHandlers.ofString()); if (schemaResponse.statusCode() >= 300) { throw new RuntimeException("Failed to fetch schema: " + schemaResponse.body()); } Schema schema = mapper.readValue(schemaResponse.body(), Schema.class); // 2) Fetch table config JSON for OFFLINE table String tableConfigUrl = String.format("%s/tables/%s?type=OFFLINE", controllerURL, sinkTableName); HttpRequest tableConfigRequest = HttpRequest.newBuilder() .uri(URI.create(tableConfigUrl)) .header("Authorization", "Basic " + encodedAuth) .GET() .build(); HttpResponse<String> tableConfigResponse = httpClient.send(tableConfigRequest, HttpResponse.BodyHandlers.ofString()); if (tableConfigResponse.statusCode() >= 300) { throw new RuntimeException("Failed to fetch table config: " + tableConfigResponse.body()); } JsonNode rootNode = mapper.readTree(tableConfigResponse.body()); JsonNode offlineNode = rootNode.get("OFFLINE"); if (offlineNode == null) { throw new RuntimeException("OFFLINE config section not found in table config response"); } TableConfig tableConfig = mapper.treeToValue(offlineNode, TableConfig.class); // 3) Fix missing ingestionConfig->batchIngestionConfig->batchConfigMaps to avoid Pinot errors during ingestion if (tableConfig.getIngestionConfig() == null) { tableConfig.setIngestionConfig(new IngestionConfig()); } IngestionConfig ingestionConfig = tableConfig.getIngestionConfig(); if (ingestionConfig.getBatchIngestionConfig() == null) { // Must provide batchConfigMaps as empty list (required) List<Map<String, String>> batchConfigMaps = new ArrayList<>(); BatchIngestionConfig batchIngestionConfig = new BatchIngestionConfig(batchConfigMaps, null, null); ingestionConfig.setBatchIngestionConfig(batchIngestionConfig); } else { BatchIngestionConfig batchIngestionConfig = ingestionConfig.getBatchIngestionConfig(); if (batchIngestionConfig.getBatchConfigMaps() == null) { batchIngestionConfig.setBatchConfigMaps(new ArrayList<>()); } } // 4) Add PinotSinkFunction to Flink DataStream using fetched schema and updated table config sinkStream.addSink( new PinotSinkFunction<>( new FlinkRowGenericRowConverter(TYPE_INFO), // Your converter according to Row types tableConfig, schema)) .name("PinotSink_" + sinkTableName) .setParallelism(PARALLELISM); } I would appreciate you guys helping us out.