slowly refactoring and exploring alternative designs, expecting tests to break for now

Author: theburningbush

README.md

@@ -46,7 +46,7 @@ This tool will *probably* work on any of the Supernote devices running the most
         snbackup -c /the/path/to/config.json
         ```  
 
-    - You can also set the environment variable `SNBACKUP_CONF` which points to the location of the **_config.json_**. This allows you to run `snbackup` from anywhere without needing to specify the config file location. The exact command to set environment variables will depend on your operating system and terminal shell.  
+    - You can also set the environment variable `SNBACKUP_CONF` which points to the location of the **_config.json_**. This allows you to run `snbackup` from anywhere without needing to specify the config file location. The exact command to set environment variables will depend on your operating system and terminal shell. Setting the environment variable will take priority over the other config lookup methods.  
         ```bash
         export SNBACKUP_CONF="/path/to/config.json"
         ```  

pyproject.toml

@@ -69,7 +69,7 @@ exclude = [
     "venv",
 ]
 
-line-length = 140
+line-length = 120
 indent-width = 4
 
 target-version = "py310"

snbackup/backup.py

@@ -7,11 +7,13 @@
 
 import httpx
 
-from .files import SnFile
+from .files import SnFile, FileMeta
+from .setup import SetupConf
 from .utilities import CustomLogger, truncate_log
 from .helpers import (
-    EXTS, FOLDERS, user_input, check_version, today_pth, load_config,
-    bytes_to_mb, count_backups, recursive_scan, locate_config
+    EXTS, FOLDERS, user_input, check_version, 
+    today_pth, load_config, bytes_to_mb,
+    count_backups, recursive_scan, locate_config
 )
 
 
@@ -77,6 +79,18 @@ def device_uri_gen(url: str, note_details: list[dict]):
             yield from device_uri_gen(url, device_data)
 
 
+def extract_meta(url: str, file_details: list[dict]):
+    """Recursive generator to extract data to instantiate SnFile object"""
+    for data in file_details:
+        if not data.get('isDirectory'):
+            # Drop the anchor slash to call joinpath later and it work properly
+            yield data.get('uri').lstrip('/'), data.get('date'), data.get('size')
+        else:
+            html = talk_to_device(url, data.get('uri').lstrip('/'))
+            re_parse = parse_html(html.text)
+            device_data = load_parsed(re_parse)
+            yield from device_uri_gen(url, device_data)
+
 # def save_file(local_pth: Path, file: bytes) -> None:
 #     """Make parent directory and write file bytes object to local disk"""
 #     local_pth.parent.mkdir(exist_ok=True, parents=True)
@@ -108,16 +122,23 @@ def previous_record_gen(json_md: Path, *, previous=None):
         previous = previous or []
 
     yield from previous
-    # for record in previous:
-    #     yield record
-        # yield (
-        #     record.get('saved_on'),
-        #     record.get('current_loc'),
-        #     record.get('uri'),
-        #     record.get('modified'),
-        #     record.get('size'),
-        #     record.get('hash')
-        # )
+
+
+def load_metadata(json_md: Path, *, meta=None) -> list:
+    """Retreive last backup metadata from file and 
+    returns the array of dictionaries or an empty array"""
+
+    try:
+        with open(json_md) as json_in:
+            meta = json.loads(json_in.read())
+    except FileNotFoundError:
+        logger.warning('Unable to locate metadata file. Creating new file')
+    except json.JSONDecodeError:
+        logger.warning('Unable to decode json in metadata file')
+    finally:
+        meta = meta or []
+    return meta
+
 
 
 def check_for_deleted(current: set, previous: set) -> list[SnFile]:
@@ -192,7 +213,6 @@ def backup() -> None:
         raise SystemExit()
 
     if args.setup:
-        from .setup import SetupConf  # Lazy import
         setup = SetupConf()
         setup.prompt()
         setup.write_config()

snbackup/files.py

@@ -1,5 +1,8 @@
+import os
 from pathlib import Path
 from hashlib import sha256
+from typing import NamedTuple
+from dataclasses import dataclass
 from datetime import datetime, date
 from functools import total_ordering
 
@@ -12,37 +15,35 @@ class BadDateError(Exception):
     """Bad datetime format or value"""
 
 
+class FileMeta(NamedTuple):
+    uri: str
+    modified: str
+    size: int
+    saved: str | None = None
+    current_loc: str | None = None
+    prev_hash: str | None = None
+
+
 @total_ordering
 class SnFile:
     """Represent an individual Supernote file"""
 
-    def __init__(
-        self, save_to: Path,
-        today: date,
-        uri: str,
-        modified: str,
-        size: int,
-        saved: str | None = None,
-        current_loc: str | None = None,
-        prev_hash: str | None = None
-    ) -> None:
+    def __init__(self, save_to: Path, today: date, file_meta: FileMeta) -> None:
         self.save_to = save_to
         self.today = today
-        self.file_uri = uri
-        self.last_modified = modified
-        self.size = size
-        self.saved = saved
-        self.current_loc = current_loc
-        self.prev_hash = prev_hash
+        self.file_meta = file_meta
         self._file_bytes = b''
+        self._calc_hash = None
+        self._consistency = None
 
     @property
     def save_date(self) -> str:
         return self.base_path.name
 
     @property
     def full_path(self) -> Path:
-        return self.base_path.joinpath(self.uri)
+        date_pth = Path(f'{self.date!s}/{self.uri}')
+        return self.save_to.joinpath(date_pth)
 
     @property
     def file_bytes(self) -> bytes:
@@ -80,13 +81,20 @@ def _calc_hash(self) -> str:
             return ''
         return sha256(self._file_bytes).hexdigest()
 
+    def _validate(self) -> None:
+        if self.prev_hash:
+            self._consistency = (self._file_hash == self.prev_hash)
+
     def _make_dir(self) -> None:
         self.full_path.parent.mkdir(exist_ok=True, parents=True)
 
     def save_file(self) -> None:
+        """More verbose write operation to ensure file is written to disk"""
         self._make_dir()
         with open(self.full_path, 'wb') as file_output:
             file_output.write(self.file_bytes)
+            file_output.flush()
+            os.fsync(file_output.fileno())
 
     def __eq__(self, other) -> bool:
         if not isinstance(other, self.__class__):
@@ -101,8 +109,27 @@ def __lt__(self, other) -> bool:
     def __hash__(self) -> int:
         return hash((self.uri, self.modified, self.size))
 
+    # def __repr__(self) -> str:
+    #     return f"{type(self).__name__}({self.base_path!r}, {self.uri!r}, '{self.modified}', {self.size})"
+
     def __repr__(self) -> str:
-        return f"{type(self).__name__}({self.base_path!r}, {self.uri!r}, '{self.modified}', {self.size})"
+        return (
+            f'{type(self).__name__}('
+            f'save_to={self.save_to!r}, '
+            f'uri={self.uri!r}, '
+            f"modified='{self.modified}', "
+            f'size={self.size}), '
+            f''
+        )
+    
+    # def __repr__(self) -> str:
+    #     return f"""{type(self).__name__}(
+    #     base_path={self.base_path!r},
+    #     uri={self.uri!r},
+    #     modified='{self.modified}',
+    #     size={self.size}
+    # )"""
+
 
     def make_record(self) -> dict:
         return {

snbackup/helpers.py

@@ -7,29 +7,55 @@
 from .setup import SetupConf
 
 FOLDERS = {
-    'note': 'Note', 
-    'document': 'Document', 
-    'export': 'EXPORT', 
-    'mystyle': 'MyStyle', 
-    'screenshot': 'SCREENSHOT', 
+    'note': 'Note',
+    'document': 'Document',
+    'export': 'EXPORT',
+    'mystyle': 'MyStyle',
+    'screenshot': 'SCREENSHOT',
     'inbox': 'INBOX',
 }
 
 EXTS = {
-    '.note', '.pdf', '.epub', '.docx', '.doc', 
-    '.txt', '.png', '.jpg', '.jpeg', '.bmp', 
-    '.webp', '.cbz', '.fb2', '.xps', '.mobi',
+    '.note',
+    '.pdf',
+    '.epub',
+    '.docx',
+    '.doc',
+    '.txt',
+    '.png',
+    '.jpg',
+    '.jpeg',
+    '.bmp',
+    '.webp',
+    '.cbz',
+    '.fb2',
+    '.xps',
+    '.mobi',
 }
 
 
 def user_input() -> Namespace:
     parser = ArgumentParser(allow_abbrev=False)
     parser.add_argument('-c', '--config', type=Path, help='Path to config.json file')
-    parser.add_argument('-f', '--full', action='store_true', help='Download all notes and files from device. Disregard any saved locally.')
-    parser.add_argument('-i', '--inspect', action='store_true', help='Inspect device for new files to download and quit')
-    parser.add_argument('-u', '--upload', nargs='+', help='Send one or more files to device. "snbackup -u file1 file2 file3"')
     parser.add_argument(
-        '-d', '--destination', default='document', type=str.lower, choices=FOLDERS, help='Destination folder to send file upload'
+        '-f',
+        '--full',
+        action='store_true',
+        help='Download all notes and files from device. Disregard any saved locally.',
+    )
+    parser.add_argument(
+        '-i', '--inspect', action='store_true', help='Inspect device for new files to download and quit'
+    )
+    parser.add_argument(
+        '-u', '--upload', nargs='+', help='Send one or more files to device. "snbackup -u file1 file2 file3"'
+    )
+    parser.add_argument(
+        '-d',
+        '--destination',
+        default='document',
+        type=str.casefold,
+        choices=FOLDERS,
+        help='Destination folder to send file upload',
     )
     parser.add_argument('-ls', '--list', action='store_true', help='List out information about backups found locally')
     parser.add_argument('-v', '--version', action='store_true', help='Print program version and quit.')
@@ -53,11 +79,7 @@ def user_input() -> Namespace:
 
 def locate_config() -> Path:
     """Look in potential config locations and return first valid"""
-    conf_locations = (
-        os.getenv('SNBACKUP_CONF', ''), 
-        SetupConf.home_conf, 
-        Path().cwd().joinpath(SetupConf.config)
-    )
+    conf_locations = (os.getenv('SNBACKUP_CONF', ''), SetupConf.home_conf, Path().cwd().joinpath(SetupConf.config))
     for conf in conf_locations:
         pth = Path(conf)
         if pth.is_file() and pth.suffix == '.json':
@@ -84,7 +106,8 @@ def today_pth(save_dir: Path) -> Path:
 
 def check_version(name: str) -> str:
     from importlib.metadata import version
-    return f"{name} v{version(name)}"
+
+    return f'{name} v{version(name)}'
 
 
 def bytes_to_mb(byte_size: int) -> str:
@@ -93,7 +116,7 @@ def bytes_to_mb(byte_size: int) -> str:
 
 
 def count_backups(directory: Path, pattern='202?-*') -> tuple[int, Path, Path]:
-    """Counts number of backup folders and returns oldest and 
+    """Counts number of backup folders and returns oldest and
     newest found on local disk"""
 
     previous = sorted(directory.glob(pattern))