Enviado em 06/12/2020 - 21:30h
Eu tenho este Dockerfile que roda e constrói uma imagem normalmente:# Base Image
FROM python:3.8-slim
LABEL maintainer="gizelly"
# Arguments that can be set with docker build
ARG AIRFLOW_VERSION=1.10.12
ARG AIRFLOW_HOME=/usr/local/airflow
# Export the environment variable AIRFLOW_HOME where airflow will be installed
ENV AIRFLOW_HOME=${AIRFLOW_HOME}
# Install dependencies and tools
RUN apt-get update -yqq && \
apt-get upgrade -yqq && \
apt-get install -yqq --no-install-recommends \
wget \
libczmq-dev \
curl \
libssl-dev \
git \
inetutils-telnet \
bind9utils freetds-dev \
libkrb5-dev \
libsasl2-dev \
libffi-dev libpq-dev \
freetds-bin build-essential \
default-libmysqlclient-dev \
apt-utils \
rsync \
zip \
unzip \
gcc \
vim \
locales \
&& apt-get clean
RUN apt-get update -y
RUN pip install deployv==0.9.173
RUN pip install psutil==5.7.3
RUN pip install python-dev-tools==2020.9.10
RUN apt-get update && apt-get install -y --no-install-recommends \
unixodbc-dev \
unixodbc \
libpq-dev
RUN python3.8 -m pip install pyodbc==4.0.30
COPY ./requirements-python3.8.txt /requirements-python3.8.txt
# Upgrade pip
# Create airflow user
# Install apache airflow with subpackages
RUN pip install --upgrade pip && \
useradd -ms /bin/bash -d ${AIRFLOW_HOME} airflow && \
pip install apache-airflow[microsoft.azure,azure_blob_storage,azure_cosmos,azure_data_lake,all_dbs,crypto,celery,postgres,kubernetes,docker]==${AIRFLOW_VERSION} --constraint /requirements-python3.8.txt
RUN python3.8 -m pip install azure-storage-blob --upgrade --force-reinstall
RUN python3.8 -m pip install azure-storage-blob --user
RUN python3.8 -m pip install azure-storage-blob --upgrade --force-reinstall
RUN python3.8 -c "from azure.storage.blob import BlobServiceClient, BlobClient, ContainerClient"
# Copy the entrypoint.sh from host to container (at path AIRFLOW_HOME)
COPY ./entrypoint.sh ./entrypoint.sh
# Set the entrypoint.sh file to be executable
RUN chmod +x ./entrypoint.sh
# Set the owner of the files in AIRFLOW_HOME to the user airflow
RUN chown -R airflow: ${AIRFLOW_HOME}
# Set the username to use
USER airflow
# Set workdir (it's like a cd inside the container)
WORKDIR ${AIRFLOW_HOME}
# Create the dags folder which will contain the DAGs
RUN mkdir dags
# Expose ports (just to indicate that this container needs to map port)
EXPOSE 8080
# Execute the entrypoint.sh
ENTRYPOINT [ "/entrypoint.sh" ]
# Editable install with no version control (apache-airflow==1.10.12)
Babel==2.8.0
Flask-Admin==1.5.4
Flask-AppBuilder==2.3.4
Flask-Babel==1.0.0
Flask-Bcrypt==0.7.1
Flask-Caching==1.3.3
Flask-JWT-Extended==3.24.1
Flask-Login==0.4.1
Flask-OpenID==1.2.5
Flask-SQLAlchemy==2.4.4
Flask-WTF==0.14.3
Flask==1.1.2
JPype1==0.7.1
JayDeBeApi==1.2.3
Jinja2==2.11.2
Mako==1.1.3
Markdown==2.6.11
MarkupSafe==1.1.1
PyHive==0.6.3
PyJWT==1.7.1
PyNaCl==1.4.0
PySmbClient==0.1.5
PyYAML==5.3.1
Pygments==2.6.1
SQLAlchemy-JSONField==0.9.0
SQLAlchemy-Utils==0.36.8
SQLAlchemy==1.3.18
Sphinx==3.2.1
Unidecode==1.1.1
WTForms==2.3.3
Werkzeug==0.16.1
adal==1.2.4
alabaster==0.7.12
alembic==1.4.2
amqp==2.6.1
analytics-python==1.2.9
ansiwrap==0.8.4
apipkg==1.5
apispec==1.3.3
appdirs==1.4.4
argcomplete==1.12.0
asn1crypto==1.4.0
astroid==2.4.2
async-generator==1.10
atlasclient==1.0.0
attrs==19.3.0
aws-sam-translator==1.26.0
aws-xray-sdk==2.6.0
azure-common==1.1.25
azure-cosmos==3.2.0
azure-datalake-store==0.0.49
azure-mgmt-containerinstance==1.5.0
azure-mgmt-resource==10.2.0
azure-nspkg==3.0.2
azure-storage-blob==2.1.0
azure-storage-common==2.1.0
azure-storage==0.36.0
backcall==0.2.0
bcrypt==3.2.0
beautifulsoup4==4.7.1
billiard==3.6.3.0
black==19.10b0
blinker==1.4
boto3==1.14.43
boto==2.49.0
botocore==1.17.43
cached-property==1.5.1
cachetools==4.1.1
cassandra-driver==3.20.2
cattrs==1.0.0
celery==4.4.7
certifi==2020.6.20
cffi==1.14.2
cfgv==3.2.0
cfn-lint==0.35.0
cgroupspy==0.1.6
chardet==3.0.4
click==6.7
cloudant==0.5.10
colorama==0.4.3
colorlog==4.0.2
configparser==3.5.3
coverage==5.2.1
croniter==0.3.34
cryptography==3.0
cx-Oracle==8.0.0
datadog==0.38.0
decorator==4.4.2
defusedxml==0.6.0
dill==0.3.2
distlib==0.3.1
dnspython==1.16.0
docker-pycreds==0.4.0
docker==3.7.3
docopt==0.6.2
docutils==0.16
ecdsa==0.14.1
elasticsearch-dsl==5.4.0
elasticsearch==5.5.3
email-validator==1.1.1
entrypoints==0.3
execnet==1.7.1
fastavro==0.24.1
filelock==3.0.12
flake8-colors==0.1.6
flake8==3.8.3
flaky==3.7.0
flask-swagger==0.2.14
flower==0.9.5
freezegun==0.3.15
fsspec==0.8.0
funcsigs==1.0.2
future-fstrings==1.2.0
future==0.18.2
gcsfs==0.6.2
google-api-core==1.22.1
google-api-python-client==1.10.0
google-auth-httplib2==0.0.4
google-auth-oauthlib==0.4.1
google-auth==1.20.1
google-cloud-bigquery==1.26.1
google-cloud-bigtable==1.4.0
google-cloud-container==1.0.1
google-cloud-core==1.4.1
google-cloud-dlp==1.0.0
google-cloud-language==1.3.0
google-cloud-secret-manager==1.0.0
google-cloud-spanner==1.17.1
google-cloud-speech==1.3.2
google-cloud-storage==1.30.0
google-cloud-texttospeech==1.0.1
google-cloud-translate==1.7.0
google-cloud-videointelligence==1.15.0
google-cloud-vision==1.0.0
google-crc32c==0.1.0
google-resumable-media==0.7.1
googleapis-common-protos==1.52.0
graphviz==0.14.1
grpc-google-iam-v1==0.12.3
grpcio-gcp==0.2.2
grpcio==1.31.0
gunicorn==20.0.4
hdfs==2.5.8
hmsclient==0.1.1
httplib2==0.18.1
humanize==2.6.0
hvac==0.10.5
identify==1.4.28
idna==2.10
imagesize==1.2.0
importlib-metadata==1.7.0
inflection==0.5.0
ipdb==0.13.3
ipython-genutils==0.2.0
ipython==7.17.0
iso8601==0.1.12
isodate==0.6.0
itsdangerous==1.1.0
jedi==0.17.2
jira==2.0.0
jmespath==0.10.0
json-merge-patch==0.2
jsondiff==1.1.2
jsonpatch==1.26
jsonpickle==1.4.1
jsonpointer==2.0
jsonschema==3.2.0
junit-xml==1.9
jupyter-client==6.1.6
jupyter-core==4.6.3
kombu==4.6.11
kubernetes==11.0.0
lazy-object-proxy==1.5.1
ldap3==2.8
lockfile==0.12.2
marshmallow-enum==1.5.1
marshmallow-sqlalchemy==0.23.1
marshmallow==2.21.0
mccabe==0.6.1
mock==4.0.2
mongomock==3.20.0
more-itertools==8.4.0
moto==1.3.14
msrest==0.6.18
msrestazure==0.6.4
multi-key-dict==2.0.3
mypy-extensions==0.4.3
mypy==0.720
mysqlclient==1.3.14
natsort==7.0.1
nbclient==0.4.1
nbformat==5.0.7
nest-asyncio==1.4.0
networkx==2.4
nodeenv==1.4.0
nteract-scrapbook==0.4.1
ntlm-auth==1.5.0
numpy==1.19.1
oauthlib==3.1.0
oscrypto==1.2.1
packaging==20.4
pandas-gbq==0.13.2
pandas==1.1.0
papermill==2.1.2
parameterized==0.7.4
paramiko==2.7.1
parso==0.7.1
pathspec==0.8.0
pbr==5.4.5
pendulum==1.4.4
pexpect==4.8.0
pickleshare==0.7.5
pinotdb==0.1.1
pluggy==0.13.1
pre-commit==2.6.0
presto-python-client==0.7.0
prison==0.1.3
prometheus-client==0.8.0
prompt-toolkit==3.0.6
protobuf==3.13.0
psutil==5.7.2
psycopg2-binary==2.8.5
ptyprocess==0.6.0
py==1.9.0
pyOpenSSL==19.1.0
pyarrow==0.17.1
pyasn1-modules==0.2.8
pyasn1==0.4.8
pycodestyle==2.6.0
pycparser==2.20
pycryptodomex==3.9.8
pydata-google-auth==1.1.0
pydruid==0.5.8
pyflakes==2.2.0
pykerberos==1.2.1
pymongo==3.10.1
pyparsing==2.4.7
pyrsistent==0.16.0
pysftp==0.2.9
pytest-cov==2.10.1
pytest-forked==1.3.0
pytest-instafail==0.4.2
pytest-rerunfailures==9.0
pytest-timeout==1.4.2
pytest-xdist==2.0.0
pytest==5.4.3
python-daemon==2.2.4
python-dateutil==2.8.1
python-editor==1.0.4
python-http-client==3.2.7
python-jenkins==1.7.0
python-jose==3.2.0
python-nvd3==0.15.0
python-slugify==4.0.1
python3-openid==3.2.0
pytz==2020.1
pytzdata==2020.1
pywinrm==0.4.1
pyzmq==19.0.2
qds-sdk==1.16.0
redis==3.5.3
regex==2020.7.14
requests-futures==0.9.4
requests-kerberos==0.12.0
requests-mock==1.8.0
requests-ntlm==1.1.0
requests-oauthlib==1.3.0
requests-toolbelt==0.9.1
requests==2.24.0
responses==0.10.16
rsa==4.6
s3transfer==0.3.3
sasl==0.2.1
sendgrid==5.6.0
sentinels==1.0.0
sentry-sdk==0.16.5
setproctitle==1.1.10
six==1.15.0
slackclient==1.3.2
snowballstemmer==2.0.0
snowflake-connector-python==2.2.10
snowflake-sqlalchemy==1.2.3
soupsieve==2.0.1
sphinx-argparse==0.2.5
sphinx-autoapi==1.0.0
sphinx-copybutton==0.3.0
sphinx-jinja==1.1.1
sphinx-rtd-theme==0.5.0
sphinxcontrib-applehelp==1.0.2
sphinxcontrib-devhelp==1.0.2
sphinxcontrib-dotnetdomain==0.4
sphinxcontrib-golangdomain==0.2.0.dev0
sphinxcontrib-htmlhelp==1.0.3
sphinxcontrib-httpdomain==1.7.0
sphinxcontrib-jsmath==1.0.1
sphinxcontrib-qthelp==1.0.3
sphinxcontrib-serializinghtml==1.1.4
sshpubkeys==3.1.0
sshtunnel==0.1.5
tabulate==0.8.7
tenacity==4.12.0
text-unidecode==1.3
textwrap3==0.9.2
thrift-sasl==0.4.2
thrift==0.13.0
toml==0.10.1
tornado==5.1.1
tqdm==4.48.2
traitlets==4.3.3
typed-ast==1.4.1
typing-extensions==3.7.4.2
tzlocal==1.5.1
unicodecsv==0.14.1
uritemplate==3.0.1
urllib3==1.25.10
vertica-python==0.11.0
vine==1.3.0
virtualenv==20.0.30
wcwidth==0.2.5
websocket-client==0.57.0
wrapt==1.12.1
xmltodict==0.12.0
yamllint==1.24.2
zdesk==2.7.1
zipp==3.1.0
zope.deprecation==4.4.0
pyodbc==4.0.30
from airflow import DAG
from datetime import datetime, timedelta
import airflow
from airflow.utils.dates import days_ago
from airflow.operators.bash_operator import BashOperator
from airflow.operators.dummy_operator import DummyOperator
# Argumentos básicos de uma DAG
default_args = {
'start_date':datetime(2020,12,3),
'owner':'precredito',
'depends_on_past': False,
'email': ['alexandre.brito@luminiitsolutions.com'],
'email_on_failure': False,
'email_on_retry': False,
'retries': 1,
'retry_delay': timedelta(seconds=5)
}
###### DAG #########
with DAG(dag_id='etl-motor-credito-verdadeiro-vitorioso.',
default_args=default_args,
schedule_interval=timedelta(minutes=10)) as dag:
start = DummyOperator(task_id='start')
etlmotor = BashOperator(task_id='blob',
bash_command="python3.8 /usr/local/airflow/dags/blob.py")
insertbd = BashOperator(task_id='azure',
bash_command="python3.8 /usr/local/airflow/dags/azure.py")
end = DummyOperator(task_id='end')
start >> etlmotor >> insertbd >> end
import pyodbc
import sys
sys.path.append(' /usr/local/lib/python3.8/site-packages')
import os, uuid
from azure.storage.blob import BlobServiceClient, BlobClient, ContainerClient
import pandas as pd
print("test")
import pyodbc
import sys
sys.path.append(' /usr/local/lib/python3.8/site-packages')
import os, uuid
from azure.storage.blob import BlobServiceClient, BlobClient, ContainerClient
import pandas as pd
print("test2")
File "/usr/local/airflow/dags/blob.py", line 5, in <module>
[2020-12-06 23:42:22,977] {bash_operator.py:157} INFO - from azure.storage.blob import BlobServiceClient
[2020-12-06 23:42:22,977] {bash_operator.py:157} INFO - File "/usr/local/airflow/dags/azure.py", line 5, in <module>
[2020-12-06 23:42:22,978] {bash_operator.py:157} INFO - from azure.storage.blob import BlobServiceClient, BlobClient, ContainerClient
[2020-12-06 23:42:22,978] {bash_operator.py:157} INFO - ModuleNotFoundError: No module named 'azure.storage'; 'azure' is not a package
[2020-12-06 23:42:22,991] {bash_operator.py:159} INFO - Command exited with return code 1
[2020-12-06 23:42:23,002] {taskinstance.py:1150} ERROR - Bash command failed
Traceback (most recent call last):
Melhorando o tempo de boot do Fedora e outras distribuições
Como instalar as extensões Dash To Dock e Hide Top Bar no Gnome 45/46
E a guerra contra bots continua
Tradução do artigo do filósofo Gottfried Wilhelm Leibniz sobre o sistema binário
Conheça o firewall OpenGFW, uma implementação do (Great Firewall of China).
Instalando o FreeOffice no LMDE 6
Anki: Remover Tags de Estilo HTML de Todas as Cartas
Colocando uma opção de redimensionamento de imagem no menu de contexto do KDE
Não consigo acessar os modos de desempenho (2)
Ubuntu — tentando iniciar o windows? (0)
[Shell Script] Script para desinstalar pacotes desnecessários no OpenSuse
[Shell Script] Script para criar certificados de forma automatizada no OpenVpn
[Shell Script] Conversor de vídeo com opção de legenda
[C/C++] BRT - Bulk Renaming Tool
[Shell Script] Criação de Usuarios , Grupo e instalação do servidor de arquivos samba