Linson Abah
06/05/2024, 6:28 PMTraceback (most recent call last):
File "/usr/local/lib/python3.10/runpy.py", line 196, in _run_module_as_main
return _run_code(code, main_globals, None,
File "/usr/local/lib/python3.10/runpy.py", line 86, in _run_code
exec(code, run_globals)
File "/usr/local/lib/python3.10/site-packages/pyflink/fn_execution/beam/beam_boot.py", line 121, in <module>
from pyflink.fn_execution.beam import beam_sdk_worker_main
File "/usr/local/lib/python3.10/site-packages/pyflink/fn_execution/beam/beam_sdk_worker_main.py", line 21, in <module>
import pyflink.fn_execution.beam.beam_operations # noqa # pylint: disable=unused-import
File "/usr/local/lib/python3.10/site-packages/pyflink/fn_execution/beam/beam_operations.py", line 20, in <module>
from apache_beam.runners import common
File "/usr/local/lib/python3.10/site-packages/apache_beam/__init__.py", line 87, in <module>
from apache_beam import coders
File "/usr/local/lib/python3.10/site-packages/apache_beam/coders/__init__.py", line 17, in <module>
from apache_beam.coders.coders import *
File "/usr/local/lib/python3.10/site-packages/apache_beam/coders/coders.py", line 59, in <module>
from apache_beam.coders import coder_impl
File "apache_beam/coders/coder_impl.py", line 57, in init apache_beam.coders.coder_impl
File "/usr/local/lib/python3.10/site-packages/fastavro/__init__.py", line 47, in <module>
import fastavro.read
File "/usr/local/lib/python3.10/site-packages/fastavro/read.py", line 4, in <module>
from . import _read_py as _read # type: ignore
File "/usr/local/lib/python3.10/site-packages/fastavro/_read_py.py", line 9, in <module>
import lzma
File "/usr/local/lib/python3.10/lzma.py", line 27, in <module>
from _lzma import *
ModuleNotFoundError: No module named '_lzma'
the job runs with any error locally, but this error comes up with I deploy it to kubernetes.
This is the dockerfile:
FROM flink:1.18.1
RUN apt-get update -y && \
apt-get install -y build-essential libssl-dev zlib1g-dev libbz2-dev libffi-dev && \
wget https://www.python.org/ftp/python/3.10.2/Python-3.10.2.tgz && \
tar -xvf Python-3.10.2.tgz && \
cd Python-3.10.2 && \
./configure --without-tests --enable-shared && \
make -j6 && \
make install && \
ldconfig /usr/local/lib && \
cd .. && rm -f Python-3.10.2.tgz && rm -rf Python-3.10.2 && \
ln -s /usr/local/bin/python3 /usr/local/bin/python && \
apt-get clean && \
rm -rf /var/lib/apt/lists/*
RUN pip3 install "apache-flink>=1.18.1,<1.19.0"
USER flink
RUN mkdir -p /opt/flink/usrlib /opt/flink/config
ADD ./lib/*.jar /opt/flink/lib
Thank you.Ryan Goldenberg
06/05/2024, 6:35 PMliblzma-dev
here's an example docker file I have
FROM flink:1.18.1
# JDK is required to install pyflink
RUN apt-get update -y && apt-get install -y openjdk-11-jdk
ENV JAVA_HOME=/usr/lib/jvm/java-11-openjdk-arm64
# Flink 1.18.1 requires Python 3.8
# Install commands taken from here
# https://nightlies.apache.org/flink/flink-docs-release-1.15/docs/deployment/resource-providers/standalone/docker/#using-flink-python-on-docker
RUN apt-get update -y && \
apt-get install -y build-essential libssl-dev zlib1g-dev libbz2-dev libffi-dev liblzma-dev jq
RUN wget https://www.python.org/ftp/python/3.8.3/Python-3.8.3.tgz && \
tar -xvf Python-3.8.3.tgz && \
cd Python-3.8.3 && \
./configure --without-tests --enable-shared && \
make -j6 && \
make install && \
ldconfig /usr/local/lib && \
cd .. && rm -f Python-3.8.3.tgz && rm -rf Python-3.8.3 && \
ln -s /usr/local/bin/python3 /usr/local/bin/python && \
apt-get clean && \
rm -rf /var/lib/apt/lists/*
# Install PyFlink
RUN python -m pip install --upgrade pip
RUN python -m pip install apache-flink==1.18.1
...Linson Abah
06/05/2024, 6:47 PMliblzma-devLinson Abah
06/05/2024, 7:14 PMRyan Goldenberg
06/05/2024, 7:27 PMliblzma-dev locally?
you could try using the dockerfile above with your jars addedLinson Abah
06/07/2024, 12:11 PM