attempt to add logic to skip an image or attachment export if their API call fails

pchang388 · pchang388 · commit 92af2826fef0 · 2024-10-31T00:46:49.000-04:00
diff --git a/bookstack_file_exporter/archiver/asset_archiver.py b/bookstack_file_exporter/archiver/asset_archiver.py
@@ -24,7 +24,7 @@ class AssetNode:
         AssetNode instance for use in other classes
     """
     def __init__(self, meta_data: Dict[str, int | str | bool]):
-        self.id: int = meta_data['id']
+        self.id_: int = meta_data['id']
         self.page_id: int = meta_data['uploaded_to']
         self.url: str = ""
         self.name: str = ""
@@ -88,7 +88,7 @@ class AttachmentNode(AssetNode):
     def __init__(self, meta_data: Dict[str, Union[int, str, bool]],
                  base_url: str):
         super().__init__(meta_data)
-        self.url: str = f"{base_url}/{self.id}"
+        self.url: str = f"{base_url}/{self.id_}"
         self.name = meta_data['name']
         log.debug("Attachment node has generated url: %s", self.url)
         self._relative_path_prefix = f"{_ATTACHMENT_DIR_NAME}"
@@ -140,7 +140,7 @@ def get_asset_nodes(self, asset_type: str) -> Dict[str, ImageNode | AttachmentNo
     def get_asset_data(self, asset_type: str,
             meta_data: Union[AttachmentNode, ImageNode]) -> Dict[str, str | bool | int | dict]:
         """Get asset data based on type"""
-        data_url = f"{self.api_urls[asset_type]}/{meta_data.id}"
+        data_url = f"{self.api_urls[asset_type]}/{meta_data.id_}"
         asset_data_response: Response = common_util.http_get_request(
             data_url,
             self._headers,
@@ -164,6 +164,7 @@ def update_asset_links(self, asset_type, page_name: str, page_data: bytes,
             asset_nodes: List[ImageNode | AttachmentNode]) -> bytes:
         """update markdown links in page data"""
         for asset_node in asset_nodes:
+            # get metadata instead of raw data/bytes
             asset_data = self.get_asset_data(asset_type, asset_node)
             asset_node.set_markdown_content(asset_data)
             if not asset_node.markdown_str:
diff --git a/bookstack_file_exporter/archiver/page_archiver.py b/bookstack_file_exporter/archiver/page_archiver.py
@@ -1,10 +1,12 @@
 from typing import Union, List, Dict
-
+import logging
 from bookstack_file_exporter.exporter.node import Node
 from bookstack_file_exporter.archiver import util as archiver_util
 from bookstack_file_exporter.archiver.asset_archiver import AssetArchiver, ImageNode, AttachmentNode
 from bookstack_file_exporter.config_helper.config_helper import ConfigNode
 
+log = logging.getLogger(__name__)
+
 _META_FILE_SUFFIX = "_meta.json"
 _TAR_SUFFIX = ".tar"
 _TAR_GZ_SUFFIX = ".tgz"
@@ -70,6 +72,16 @@ def archive_pages(self, page_nodes: Dict[int, Node]):
                 page_images = image_nodes[page.id_]
             if page.id_ in attachment_nodes:
                 page_attachments = attachment_nodes[page.id_]
+            failed_images = self.archive_page_assets("images", page.parent.file_path,
+                                     page.name, page_images)
+            failed_attach = self.archive_page_assets("attachments", page.parent.file_path,
+                                     page.name, page_attachments)
+            if failed_images:
+                # exclude from page_images so it doesn't attempt to get modified in markdown file 
+                page_images = [img for img in page_images if img.id_ not in failed_images]
+            if failed_attach:
+                # exclude from page_attachments so it doesn't attempt to get modified in markdown file
+                page_attachments = [attach for attach in page_attachments if attach.id_ not in failed_attach]
             for export_format in self.export_formats:
                 page_data = self._get_page_data(page.id_, export_format)
                 if page_images and export_format == 'markdown':
@@ -80,10 +92,6 @@ def archive_pages(self, page_nodes: Dict[int, Node]):
                                                       page_data, page_attachments)
                 self._archive_page(page, export_format,
                                     page_data)
-            self.archive_page_assets("images", page.parent.file_path,
-                                     page.name, page_images)
-            self.archive_page_assets("attachments", page.parent.file_path,
-                                     page.name, page_attachments)
             if self.asset_config.export_meta:
                 self._archive_page_meta(page.file_path, page.meta)
 
@@ -123,15 +131,26 @@ def _modify_markdown(self, asset_type: str,
                                         asset_nodes)
 
     def archive_page_assets(self, asset_type: str, parent_path: str, page_name: str,
-                            asset_nodes: List[ImageNode | AttachmentNode]):
+                            asset_nodes: List[ImageNode | AttachmentNode]) -> Dict[int, int]:
         """pull images locally into a directory based on page"""
         if not asset_nodes:
-            return
+            return {}
+        # use a map for faster lookup
+        failed_assets = {}
         node_base_path = f"{self.archive_base_path}/{parent_path}/"
         for asset_node in asset_nodes:
-            asset_data = self.asset_archiver.get_asset_bytes(asset_type, asset_node.url)
+            try:
+                asset_data = self.asset_archiver.get_asset_bytes(asset_type, asset_node.url)
+            except:
+                # probably unnecessary, but just in case
+                if asset_node.id_ not in failed_assets:
+                    failed_assets[asset_node.id_] = 0
+                # a 404 or other error occurred, skip this asset, already logged in http request exception
+                log.error(f"Failed to get image or attachment data for asset located at: {asset_node.url} - skipping")
+                continue
             asset_path = f"{node_base_path}/{asset_node.get_relative_path(page_name)}"
             self.write_data(asset_path, asset_data)
+        return failed_assets
 
     def write_data(self, file_path: str, data: bytes):
         """write data to a tar file
diff --git a/bookstack_file_exporter/exporter/node.py b/bookstack_file_exporter/exporter/node.py
@@ -38,6 +38,7 @@ def __init__(self, meta: Dict[str, Union[str, int]],
         # for convenience/usage for exporter
         # self.name: str = self.meta['slug']
         self.name = self.get_name(self.meta['slug'], self.meta['name'])
+        # id() is a built-in function and should not be used as a variable name
         self.id_: int = self.meta['id']
         self._display_name = self.meta['name']
         # children
diff --git a/bookstack_file_exporter/run.py b/bookstack_file_exporter/run.py
@@ -53,4 +53,5 @@ def exporter(args: argparse.Namespace):
     # clean up the .tgz archive since it is already uploaded
     archive.clean_up()
 
+    log.info(f"Created file archive: {archive.archive_dir}.tgz")
     log.info("Completed run")
