Commit 77039e0d authored by Timothy Ardha's avatar Timothy Ardha

Add and update mgate's zip and delete function

parent 4c019de7
...@@ -159,10 +159,10 @@ with DAG("APJ_1_mgate", ...@@ -159,10 +159,10 @@ with DAG("APJ_1_mgate",
op_kwargs={}, op_kwargs={},
) )
remove_file_before = BashOperator( delete_before = BashOperator(
task_id="remove_file_before", task_id="delete_before",
bash_command=f""" bash_command=f"""
cd {Variable.get("LOCAL_PATH")}{DS_FOLDER}/{yesterday_lusa}/ && rm {yesterday_lusa}* cd {Variable.get("LOCAL_PATH")}{DS_FOLDER}/ && rm {yesterday_lusa}.zip
""" """
) )
...@@ -266,10 +266,17 @@ with DAG("APJ_1_mgate", ...@@ -266,10 +266,17 @@ with DAG("APJ_1_mgate",
""", """,
) )
delete_before = BashOperator( zip_today = BashOperator(
task_id="zip_today",
bash_command=f"""
cd /opt/airflow/dags/DFE/{DS_FOLDER}/ && zip -r {yesterday_nodash}.zip {yesterday_nodash};
"""
)
delete_today = BashOperator(
task_id="delete_before", task_id="delete_before",
bash_command=f""" bash_command=f"""
rm -rf /opt/airflow/dags/DFE/{DS_FOLDER}/{yesterday_lusa}; rm -rf /opt/airflow/dags/DFE/{DS_FOLDER}/{yesterday_nodash};
""", """,
) )
...@@ -285,4 +292,4 @@ with DAG("APJ_1_mgate", ...@@ -285,4 +292,4 @@ with DAG("APJ_1_mgate",
begin >> check_done_ext >> remove_file_before >> history_start >> create_folder >> ds_list_extractor >> ds_get_syntax >> ds_table_to_csv >> ds_get_ddl >> pg_drop_schema >> pg_create_schema >> pg_ddl_syntax >> pg_create_table >> pg_push_syntax >> pg_csv_to_table >> set_access_schemma >> set_access_all_table >> ds_ext_done >> delete_before >> history_finish begin >> check_done_ext >> delete_before >> history_start >> create_folder >> ds_list_extractor >> ds_get_syntax >> ds_table_to_csv >> ds_get_ddl >> pg_drop_schema >> pg_create_schema >> pg_ddl_syntax >> pg_create_table >> pg_push_syntax >> pg_csv_to_table >> set_access_schemma >> set_access_all_table >> ds_ext_done >> zip_today >> delete_today >> history_finish
\ No newline at end of file \ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment