Commit c348553c authored by Timothy Ardha's avatar Timothy Ardha

Menambahkan function zip file

parent 6b6903e4
import json, os import json, os
import shutil
from datetime import date from datetime import date
import calendar import calendar
...@@ -24,6 +25,7 @@ from airflow import XComArg ...@@ -24,6 +25,7 @@ from airflow import XComArg
yesterday_nodash = (datetime.now() - timedelta(1)).strftime('%Y%m%d') if Variable.get( yesterday_nodash = (datetime.now() - timedelta(1)).strftime('%Y%m%d') if Variable.get(
"DATE_OF_DATA") == 'today' else Variable.get("DATE_OF_DATA") "DATE_OF_DATA") == 'today' else Variable.get("DATE_OF_DATA")
yesterday_strip = datetime.strptime(yesterday_nodash, '%Y%m%d').strftime('%Y-%m-%d') yesterday_strip = datetime.strptime(yesterday_nodash, '%Y%m%d').strftime('%Y-%m-%d')
yesterday_lusa = (datetime.strptime(yesterday_nodash, '%Y%m%d') - timedelta(1))
d = f"""{datetime.now().strftime('%Y-%m-%d %H:%M:%S')}""" d = f"""{datetime.now().strftime('%Y-%m-%d %H:%M:%S')}"""
today = d[:] today = d[:]
...@@ -135,6 +137,11 @@ def pg_push_csv(ti, copy_sql, file_id): ...@@ -135,6 +137,11 @@ def pg_push_csv(ti, copy_sql, file_id):
f"""{yesterday_nodash}/{yesterday_nodash}.""" if Variable.get( f"""{yesterday_nodash}/{yesterday_nodash}.""" if Variable.get(
"WITH_DATE") == 'Y' else '') + f"""{file_id}.csv""") "WITH_DATE") == 'Y' else '') + f"""{file_id}.csv""")
def to_zip():
path_folder = f"{Variable.get("LOCAL_PATH")}{yesterday_lusa}"
shutil.make_archive(path_folder, 'zip', path_folder)
shutil.rmtree(path_folder)
with DAG("APJ_1_mgate", with DAG("APJ_1_mgate",
start_date=datetime(2021, 1, 1), start_date=datetime(2021, 1, 1),
schedule_interval='10 0 * * *', schedule_interval='10 0 * * *',
...@@ -266,7 +273,12 @@ with DAG("APJ_1_mgate", ...@@ -266,7 +273,12 @@ with DAG("APJ_1_mgate",
task_id="history_finish", task_id="history_finish",
postgres_conn_id=POSTGRES_CONN_ID, postgres_conn_id=POSTGRES_CONN_ID,
) )
to_zip = PythonOperator(
task_id='to_zip',
python_callable=to_zip
)
begin >> check_done_ext >> history_start >> create_folder >> ds_list_extractor >> ds_get_syntax >> ds_table_to_csv >> ds_get_ddl >> pg_drop_schema >> pg_create_schema >> pg_ddl_syntax >> pg_create_table >> pg_push_syntax >> pg_csv_to_table >> set_access_schemma >> set_access_all_table >> ds_ext_done >> history_finish begin >> check_done_ext >> history_start >> create_folder >> ds_list_extractor >> ds_get_syntax >> ds_table_to_csv >> ds_get_ddl >> pg_drop_schema >> pg_create_schema >> pg_ddl_syntax >> pg_create_table >> pg_push_syntax >> pg_csv_to_table >> set_access_schemma >> set_access_all_table >> ds_ext_done >> history_finish >> to_zip
\ No newline at end of file \ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment