Create airflow user
groupadd airflow
useradd airflow -g airflow
Install python3
yum -y update
yum -y install gcc -- will be used later to install mysqlclient
yum -y install gcc-objc gcc-objc + + libobjc -- does not install and reports error gcc: error trying to exec 'cc1plus': execvp: No such file or directory
yum -y install cyrus-sasl cyrus-sasl-devel cyrus-sasl-lib -- error if not installed sasl/saslwrapper.h:22:23: fatal error: sasl/sasl.h: No such file or directory
yum install -y python3 python3-devel rsync MySQL-python mysql-devel
yum install -y cyrus-sasl-plain cyrus-sasl-devel cyrus-sasl-gssapi cyrus-sasl-md5
#Install python3 virtual environment
sudo su-airflow
python3 -m venv venv
Install airflow2
source venv/bin/activate # Note: refer to other airflow required packages and versions for installation
pip install acryl-datahub==0.8.38.3
pip install alembic==1.7.7
pip install aliyun-python-sdk-core==2.13.36
pip install aliyun-python-sdk-kms==2.15.0
pip install amqp==5.1.1
pip install anyio==3.6.1
pip install apache-airflow==2.2.5
pip install apache-airflow-providers-alibaba==1.0.1
pip install apache-airflow-providers-apache-hdfs==2.2.1
pip install apache-airflow-providers-apache-hive==2.3.0
pip install apache-airflow-providers-apache-spark 2.1.1
pip install apache-airflow-providers-celery 2.1.0
pip install apache-airflow-providers-dingding 2.0.2
pip install apache-airflow-providers-ftp==2.1.0
pip install apache-airflow-providers-http 2.1.0
pip install apache-airflow-providers-imap 2.2.1
pip install apache-airflow-providers-mysql==2.2.1
pip install apache-airflow-providers-redis==2.0.2
pip install apache-airflow-providers-sqlite 2.1.1
pip install apispec==3.3.2
pip install argcomplete==2.0.0
pip install async-generator==1.10
pip install async-timeout==4.0.2
pip install attrs==20.3.0
pip install avro==1.10.2
pip install avro-gen3==0.7.4
pip install Babel==2.10.3
pip install backports.zoneinfo==0.2.1
pip install billiard==3.6.4.0
pip install blinker==1.4
pip install cached-property==1.5.2
pip install cachelib==0.6.0
pip install cachetools==4.2.4
pip install cattrs==1.0.0
pip install celery==5.1.2
pip install certifi==2022.6.15
pip install cffi==1.15.0
pip install charset-normalizer==2.0.12
pip install click==7.1.2
pip install click-default-group==1.2.2
pip install click-didyoumean==0.3.0
pip install click-plugins==1.1.1
pip install click-repl==0.2.0
pip install clickclick==20.10.2
pip install cligj==0.7.2
pip install colorama==0.4.5
pip install colorlog==6.6.0
pip install commonmark==0.9.1
pip install connexion==2.13.1
pip install contextvars==2.4
pip install crcmod==1.7
pip install croniter==1.3.5
pip install cryptography==37.0.2
pip install dataclasses==0.8
pip install decorator==5.1.1
pip install defusedxml==0.7.1
pip install Deprecated==1.2.13
pip install dill==0.3.4
pip install DingtalkChatbot==1.5.3
pip install dnspython==2.2.1
pip install docker==5.0.3
pip install docopt==0.6.2
pip install docutils==0.16
pip install email-validator==1.2.1
pip install entrypoints==0.4
pip install et-xmlfile==1.1.0
pip install expandvars==0.9.0
pip install fastavro==1.4.7
pip install Fiona==1.8.21
pip install Flask==1.1.4
pip install Flask-AppBuilder==3.4.5
pip install Flask-Babel==2.0.0
pip install Flask-Caching==1.10.1
pip install Flask-JWT-Extended==3.25.1
pip install Flask-Login==0.4.1
pip install Flask-OpenID==1.3.0
pip install Flask-Session==0.4.0
pip install Flask-SQLAlchemy==2.5.1
pip install Flask-WTF==0.14.3
pip install flower==1.0.0
pip install future==0.18.2
pip install geopandas==0.9.0
pip install google-auth==2.8.0
pip install graphviz==0.19.1
pip install gssapi==1.7.3
pip install gunicorn==20.1.0
pip install h11==0.12.0
pip install hdfs==2.7.0
pip install hmsclient==0.1.1
pip install httpcore==0.14.7
pip install httpx==0.22.0
pip install humanfriendly==10.0
pip install humanize==3.14.0
pip install idna==3.3
pip install immutables==0.18
pip install importlib-metadata==4.8.3
pip install importlib-resources==5.4.0
pip install inflection==0.5.1
pip install iso8601==1.0.2
pip install itsdangerous==1.1.0
pip install Jinja2==2.11.3
pip install jmespath==0.10.0
pip install joblib==1.1.0
pip install jsonschema==3.2.0
pip install kafka==1.3.5
pip install kafka-python==2.0.2
pip install kombu==5.1.0
pip install krb5==0.3.0
pip install kubernetes==23.6.0
pip install lazy-object-proxy==1.7.1
pip install lockfile==0.12.2
pip install Mako==1.1.6
pip install Markdown==3.3.7
pip install MarkupSafe==2.0.1
pip install marshmallow==3.14.1
pip install marshmallow-enum==1.5.1
pip install marshmallow-oneofschema==3.0.1
pip install marshmallow-sqlalchemy==0.26.1
pip install mixpanel==4.9.0
pip install munch==2.5.0
pip install mypy-extensions==0.4.3
pip install mysql-connector-python==8.0.29
pip install mysqlclient==2.1.0
pip install numpy==1.19.5
pip install oauthlib==3.2.0
pip install openpyxl==3.0.10
pip install oss2==2.15.0
pip install packaging==21.3
pip install pandas==1.1.5
pip install patsy==0.5.2
pip install pendulum==2.1.2
pip install pep562==1.1
pip install pika==1.3.1
pip install pip==21.3.1
pip install prison==0.2.1
pip install progressbar2==3.55.0
pip install prometheus-client==0.14.1
pip install prompt-toolkit==3.0.29
pip install protobuf==3.19.4
pip install psutil==5.9.1
pip install psycopg2==2.7.7
pip install psycopg2-binary==2.9.3
pip install pure-sasl==0.6.2
pip install py4j==0.10.9.3
pip install pyarrow==6.0.1
pip install pyasn1==0.4.8
pip install pyasn1-modules==0.2.8
pip install pycparser==2.21
pip install pycryptodome==3.14.1
pip install pydantic==1.9.1
pip install Pygments==2.12.0
pip install PyHive==0.6.5
pip install PyJWT==1.7.1
pip install pymongo==4.1.1
pip install PyMySQL==1.0.2
pip install pyparsing==3.0.9
pip install pyproj==3.0.1
pip install pyrsistent==0.18.0
pip install pyspark==3.2.1
pip install pyspnego==0.5.2
pip install python-daemon==2.3.0
pip install python-dateutil==2.8.2
pip install python-nvd3==0.15.0
pip install python-slugify==4.0.1
pip install python-utils==3.3.3
pip install python3-openid==3.2.0
pip install pytz==2022.1
pip install pytz-deprecation-shim==0.1.0.post0
pip install pytzdata==2020.1
pip install PyYAML==6.0
pip install ratelimiter==1.2.0.post0
pip install redis==3.5.3
pip install requests==2.27.1
pip install requests-kerberos==0.14.0
pip install requests-oauthlib==1.3.1
pip install rfc3986==1.5.0
pip install rich==12.4.4
pip install rsa==4.8
pip install sasl==0.3.1
pip install scikit-learn==0.24.2
pip install scipy==1.5.4
pip install setproctitle==1.2.3
pip install setuptools==39.2.0
pip install Shapely==1.8.2
pip install six==1.16.0
pip install sklearn==0.0
pip install snakebite-py3==3.0.5
pip install sniffio==1.2.0
pip install SQLAlchemy==1.3.24
pip install SQLAlchemy-JSONField==1.0.0
pip install SQLAlchemy-Utils==0.38.2
pip install stackprinter==0.2.6
pip install statsd==4.0.1
pip install statsmodels==0.12.2
pip install supervisor==4.2.4
pip install swagger-ui-bundle==0.0.9
pip install tabulate==0.8.10
pip install tenacity==8.0.1
pip install termcolor==1.1.0
pip install text-unidecode==1.3
pip install threadpoolctl==3.1.0
pip install thrift==0.16.0
pip install thrift-sasl==0.4.3
pip install toml==0.10.2
pip install tornado==6.1
pip install tqdm==4.64.0
pip install types-Deprecated==1.2.8
pip install types-termcolor==1.1.4
pip install typing==3.7.4.3
pip install typing_extensions==4.1.1
pip install typing-inspect==0.7.1
pip install tzdata==2022.1
pip install tzlocal==4.2
pip install ufile==3.2.6
pip install unicodecsv==0.14.1
pip install urllib3==1.26.9
pip install vine==5.0.0
pip install wcwidth==0.2.5
pip install websocket-client==1.3.1
pip install Werkzeug==1.0.1
pip install wrapt==1.14.1
pip install WTForms==2.3.3
pip install xgboost==1.5.2
pip install zipp==3.6.0
#initialization
Modify airflow configuration airflow.cfg #It is recommended to refer to other airflow documents directly
airflow.cfg modification
/home/airflow/airflow/plugins -> /home/airflow/airflow/airflow-bigdata/plugins
dags_folder = /home/airflow/airflow/dags -> dags_folder = /home/airflow/airflow/airflow-bigdata/dags
sql_alchemy_conn = mysql://userName:password@mysqlhost/database?charset=utf8
airflow #initialization
airflow db init
If an error is reported: Exceptobal variable explicit_defaults_for_timestamp needs to be on (1) for mysql
Need to modify mysql's explicit_defaults_for_timestamp
vim /etc/my.cnf
Added: explicit_defaults_for_timestamp=true
Restart mysql: service mysql restart
airflow users create \
--username admin \
--firstname admin \
--lastname company_name \
--role Admin \
--email [email protected]
passwd
Install redis on the database node. If it is already installed, it can be ignored
#Install redis
yum install redis
#Start redis
systemctl start redis
#Enable redis to start automatically at boot
systemctl enable redis
Configuring CeleryExecutor
executor = CeleryExecutor
broker_url = redis://127.0.0.1:6379/0
/result_backend
= db + mysql://root:[email protected]:3306/airflow_db?charset=utf8
Process management
pip install supervisor
mkdir venv/etc
echo_supervisord_conf > venv/etc/supervisord.conf
Configuration:
[program:airflow_web]
command=airflow webserver -p 8080
autostart=true
autorestart=true
stopasgroup=true
killasgroup=true
stdout_logfile=/home/airflow/airflow/webserver.log
[program:airflow_scheduler]
command=airflow scheduler
autorestart=true
stopasgroup=true
killasgroup=true
stdout_logfile=/home/airflow/airflow/scheduler.log
[program:airflow_worker]
command=airflowceleryworker
autostart=true
autorestart=true
stopasgroup=true
killasgroup=true
environment=C_FORCE_home/airflow=true
stdout_logfile=/home/airflow/airflow/worker.log
[program:airflow_flower]
command=airflow celery flower --basic-auth=admin:admin
autostart=true
autorestart=true
stopasgroup=true
killasgroup=true
stdout_logfile=/home/airflow/airflow/flower.log
#Start process management
supervisord -c venv/etc/supervisord.conf
Add environment variables
Add environment variables in /etc/profile
export PYTHONPATH=/home/airflow/airflow/airflow-bigdata:$PYTHONPATH