From f16b3ae07827787c48d1fd5ab4deff7081f14d08 Mon Sep 17 00:00:00 2001 From: Shane Frischkorn Date: Fri, 4 Feb 2022 14:59:33 +1000 Subject: [PATCH 1/5] Updating base image to a more secure one --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 9cb2276..67a36ab 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.8 +FROM python:3.8-slim RUN pip install ofxparse RUN pip install watchdog From 56d4354b53351ec09682fe08180b737ee6efdf55 Mon Sep 17 00:00:00 2001 From: Shane Frischkorn Date: Mon, 7 Feb 2022 12:48:28 +1000 Subject: [PATCH 2/5] Added extra sleep time to wait for download --- converter.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/converter.py b/converter.py index d8ed547..72d593e 100644 --- a/converter.py +++ b/converter.py @@ -134,16 +134,17 @@ class Handler(watchdog.events.PatternMatchingEventHandler): fileExists = False timeout = 0 + while not fileExists: fileExists = os.path.isfile(event.src_path) - time.sleep(1) + time.sleep(5) timeout += 1 if timeout > 60: logging.error('Timeout waiting for file {} to exist. Aborting processing.'.format(event.src_path)) return - historicalSize = -1 + while (historicalSize != os.path.getsize(event.src_path)): logging.info('waiting for copy to finish....') historicalSize = os.path.getsize(event.src_path) From 1205dddc9ec5f18d412a86496030716f5c92a6bd Mon Sep 17 00:00:00 2001 From: Shane Frischkorn Date: Fri, 11 Feb 2022 11:14:00 +1000 Subject: [PATCH 3/5] Fixing bug --- converter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/converter.py b/converter.py index 72d593e..0dde5c0 100644 --- a/converter.py +++ b/converter.py @@ -144,7 +144,7 @@ class Handler(watchdog.events.PatternMatchingEventHandler): logging.error('Timeout waiting for file {} to exist. Aborting processing.'.format(event.src_path)) return - + historicalSize = -1 while (historicalSize != os.path.getsize(event.src_path)): logging.info('waiting for copy to finish....') historicalSize = os.path.getsize(event.src_path) From ed85763ff5b2d55e15546758cddbb9866b95e967 Mon Sep 17 00:00:00 2001 From: Shane Frischkorn Date: Fri, 23 Sep 2022 11:59:57 +1000 Subject: [PATCH 4/5] Added some debugging --- converter.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/converter.py b/converter.py index 0dde5c0..883141d 100644 --- a/converter.py +++ b/converter.py @@ -178,6 +178,10 @@ class Handler(watchdog.events.PatternMatchingEventHandler): path.replace(destination) logging.info("Processing successfully finished for {}".format(event.src_path)) + + def on_created(self, event): + logging.info('{} on {}'.format(event.event_type, event.src_path)) + if __name__ == "__main__": event_handler = Handler() From 9ce052e04dd2aa21d06a37fd096a52ecea1ab36b Mon Sep 17 00:00:00 2001 From: Shane Frischkorn Date: Fri, 23 Sep 2022 15:06:20 +1000 Subject: [PATCH 5/5] Updating converter to pick up files copied in by nextcloud --- converter.py | 45 +++++++++++++++++++++++++-------------------- 1 file changed, 25 insertions(+), 20 deletions(-) diff --git a/converter.py b/converter.py index 883141d..378edaa 100644 --- a/converter.py +++ b/converter.py @@ -153,34 +153,39 @@ class Handler(watchdog.events.PatternMatchingEventHandler): logging.info("file copy has now finished") with open(event.src_path, 'r') as file: - qfx = OfxParser.parse(file, fail_fast=False) - statement, acct_name = Handler.get_statement_from_qfx(qfx) + try: + qfx = OfxParser.parse(file, fail_fast=False) + statement, acct_name = Handler.get_statement_from_qfx(qfx) - path = Path(event.src_path) - path.resolve() + path = Path(event.src_path) + path.resolve() - converted_dir = path.parent / CONVERTED_DIR - if not converted_dir.exists(): - converted_dir.mkdir() + converted_dir = path.parent / CONVERTED_DIR + if not converted_dir.exists(): + converted_dir.mkdir() - out_file = str(path.parent / CONVERTED_DIR / (acct_name + '-' + qfx.signon.dtserver + '.csv')) - Handler.write_csv(statement, out_file) + out_file = str(path.parent / CONVERTED_DIR / (acct_name + '-' + qfx.signon.dtserver + '.csv')) + Handler.write_csv(statement, out_file) - #Now move the input file to backup - archive_file_dir = path.parent / BACKUP_DIR - archive_file = (path.stem + '{:04d}' + path.suffix) - destination = Handler.unique_path(archive_file_dir, archive_file) + #Now move the input file to backup + archive_file_dir = path.parent / BACKUP_DIR + archive_file = (path.stem + '{:04d}' + path.suffix) + destination = Handler.unique_path(archive_file_dir, archive_file) - if not archive_file_dir.exists(): - archive_file_dir.mkdir() + if not archive_file_dir.exists(): + archive_file_dir.mkdir() - if not destination.exists(): - path.replace(destination) + if not destination.exists(): + path.replace(destination) + except: + logging.info("Failed to process {}".format(event.src_path)) logging.info("Processing successfully finished for {}".format(event.src_path)) - - def on_created(self, event): - logging.info('{} on {}'.format(event.event_type, event.src_path)) + + def on_modified(self, event): + logging.info('Found modified file: {}'.format(event.src_path)) + self.on_created(event) + if __name__ == "__main__":