Update backup.py

Removed legacy chunked upload method 'put_stream' in favor of 'put_file' for better proxy compatibility.

CHANGELOG.md

@@ -4,6 +4,21 @@ All notable changes to this project will be documented in this file.
 
 The format is based on Keep a Changelog, and this project adheres to Semantic Versioning.
 
+## [0.2.0] - 2026-01-14
+### Added
+- Improved cross-version compatibility with Home Assistant backup metadata by normalizing backup schema fields (e.g., `addons`, `database_included`, etc.).
+- More robust metadata serialization for `AgentBackup` across Home Assistant versions (supports different serialization methods).
+
+### Fixed
+- Improved upload reliability by spooling backup streams to a temporary file and uploading with Content-Length (avoids chunked WebDAV uploads that may cause reverse proxy 504 timeouts).
+- Added non-restrictive client timeouts for long-running WebDAV operations to prevent client-side aborts.
+- Fixed backup listing failures caused by missing expected metadata keys in different Home Assistant versions.
+
+## [0.1.1-alpha] - 2026-01-14
+### Fixed
+- Improved upload reliability by spooling backup streams to a temporary file and uploading with Content-Length (avoids chunked WebDAV uploads that may cause reverse proxy 504 timeouts).
+- Set a non-restrictive client timeout for WebDAV PUT requests to prevent client-side premature aborts on slow connections.
+
 ## [0.1.0-alpha] - 2026-01-14
 ### Added
 - Initial alpha release

README.md

@@ -1,4 +1,4 @@
-# Owncloud Backup Homeassistant by René Bachmann
+# Owncloud Backup Homeassistant
 
 Home Assistant custom integration that adds **ownCloud (Classic/Server)** as a **Backup Location / Backup Agent** using the official **WebDAV** interface.
 
@@ -8,8 +8,8 @@ This integration allows you to:
 - download and **restore** backups via the Home Assistant UI
 - authenticate using **either** an ownCloud **App Password** (recommended for 2FA) **or** the regular account password
 
-> **Status:** `0.1.0-alpha`  
-> This is an early alpha release. Please test on a non-critical system first.
+> **Status:** `0.2.0`  
+> This release focuses on reliability and compatibility across Home Assistant versions.
 
 ---
 
@@ -28,6 +28,15 @@ This integration allows you to:
 - ✅ English UI & documentation
 - ✅ HACS-ready repository structure
 
+### Upload reliability (important)
+To improve reliability behind reverse proxies and avoid WebDAV timeouts with chunked uploads,
+the integration **spools the backup to a temporary file** and then uploads it with a proper
+**Content-Length** header.
+
+### Home Assistant compatibility
+Home Assistant has evolved its backup metadata schema over time. This integration normalizes
+backup metadata keys to remain compatible across multiple Home Assistant versions.
+
 ---
 
 ## Requirements
@@ -101,6 +110,15 @@ Home Assistant will download the `.tar` from ownCloud using the Backup Agent API
 
 ## Troubleshooting
 
+### "Upload failed" / HTTP 504 (Gateway Timeout)
+A 504 typically indicates a reverse proxy / gateway timeout (e.g., Nginx/Traefik/Cloudflare).
+This integration uploads with Content-Length (non-chunked) for better compatibility.
+
+If you still see 504:
+- Increase proxy timeouts (e.g. `proxy_read_timeout`, `proxy_send_timeout` in Nginx)
+- Ensure large uploads are allowed (`client_max_body_size` in Nginx)
+- Avoid buffering restrictions for WebDAV endpoints
+
 ### "Cannot connect"
 - Check your **Base URL**
 - Make sure the ownCloud user can access WebDAV
@@ -112,12 +130,6 @@ Home Assistant will download the `.tar` from ownCloud using the Backup Agent API
   - install the CA properly, or
   - temporarily disable **Verify SSL** (not recommended for production)
 
-### Missing backups in list
-- Ensure the configured backup folder is correct
-- Check that ownCloud contains either:
-  - `.json` metadata files (preferred), or
-  - `.tar` files (fallback)
-
 ---
 
 ## Security notes

custom_components/owncloud_backup/backup.py

@@ -3,7 +3,10 @@ from __future__ import annotations
 import asyncio
 import json
 import logging
+import os
+import tempfile
 from collections.abc import AsyncIterator, Callable, Coroutine
+from dataclasses import asdict, is_dataclass
 from typing import Any
 
 from homeassistant.components.backup import (
@@ -12,7 +15,6 @@ from homeassistant.components.backup import (
     BackupAgentError,
     BackupNotFound,
 )
-
 from homeassistant.core import HomeAssistant, callback
 
 from .const import (
@@ -20,6 +22,7 @@ from .const import (
     DATA_CLIENT,
     DOMAIN,
     META_SUFFIX,
+    SPOOL_FLUSH_BYTES,
     TAR_PREFIX,
     TAR_SUFFIX,
 )
@@ -36,14 +39,112 @@ def _make_meta_name(backup_id: str) -> str:
     return f"{TAR_PREFIX}{backup_id}{META_SUFFIX}"
 
 
+def _normalize_backup_dict(d: dict[str, Any]) -> dict[str, Any]:
+    """Normalize backup metadata to satisfy multiple HA schema versions."""
+    d = dict(d)
+
+    # Identity fields (varies by HA versions)
+    d.setdefault("backup_id", d.get("slug", ""))
+    d.setdefault("slug", d.get("backup_id", ""))
+
+    # Presentation fields
+    d.setdefault("name", f"ownCloud backup ({d.get('backup_id') or d.get('slug') or 'unknown'})")
+    d.setdefault("date", d.get("created_at", ""))
+    d.setdefault("size", 0)
+    d.setdefault("protected", False)
+    d.setdefault("compressed", True)
+    d.setdefault("extra_metadata", {})
+
+    # Content selections
+    d.setdefault("addons", [])
+    d.setdefault("folders", [])
+
+    # Older schema booleans
+    d.setdefault("database", True)
+    d.setdefault("homeassistant", True)
+
+    # Newer schema booleans
+    d.setdefault("database_included", d.get("database", True))
+    d.setdefault("homeassistant_included", d.get("homeassistant", True))
+    d.setdefault("addons_included", bool(d.get("addons", [])))
+    d.setdefault("folders_included", bool(d.get("folders", [])))
+
+    # Keep old keys consistent
+    d["database"] = bool(d.get("database_included", True))
+    d["homeassistant"] = bool(d.get("homeassistant_included", True))
+
+    if not isinstance(d.get("addons"), list):
+        d["addons"] = []
+    if not isinstance(d.get("folders"), list):
+        d["folders"] = []
+
+    return d
+
+
+def _agentbackup_to_dict(backup: AgentBackup) -> dict[str, Any]:
+    """Serialize AgentBackup in a HA-version-independent way."""
+    if hasattr(backup, "to_dict"):
+        raw = backup.to_dict()  # type: ignore[assignment]
+    elif hasattr(backup, "as_dict"):
+        raw = backup.as_dict()  # type: ignore[assignment]
+    elif is_dataclass(backup):
+        raw = asdict(backup)
+    else:
+        raw = {k: v for k, v in vars(backup).items() if not k.startswith("_")}
+
+    return _normalize_backup_dict(raw)
+
+
 def _agentbackup_from_dict(d: dict[str, Any]) -> AgentBackup:
-    """Best-effort create AgentBackup across HA versions."""
+    """Create AgentBackup from dict and ensure required keys exist."""
+    d = _normalize_backup_dict(d)
+
     from_dict = getattr(AgentBackup, "from_dict", None)
     if callable(from_dict):
         return from_dict(d)  # type: ignore[misc]
     return AgentBackup(**d)  # type: ignore[arg-type]
 
 
+async def _spool_stream_to_tempfile(stream: AsyncIterator[bytes]) -> tuple[str, int]:
+    """Spool an async byte stream into a temporary file and return (path, size)."""
+    fd, path = tempfile.mkstemp(prefix="owncloud_backup_", suffix=".tar")
+    os.close(fd)
+
+    size = 0
+    buf = bytearray()
+
+    try:
+        async for chunk in stream:
+            if not chunk:
+                continue
+            buf.extend(chunk)
+            size += len(chunk)
+
+            if len(buf) >= SPOOL_FLUSH_BYTES:
+                data = bytes(buf)
+                buf.clear()
+                await asyncio.to_thread(_write_bytes_to_file, path, data, append=True)
+
+        if buf:
+            await asyncio.to_thread(_write_bytes_to_file, path, bytes(buf), append=True)
+
+        return path, size
+
+    except Exception:
+        try:
+            os.remove(path)
+        except OSError:
+            pass
+        raise
+
+
+def _write_bytes_to_file(path: str, data: bytes, *, append: bool) -> None:
+    mode = "ab" if append else "wb"
+    with open(path, mode) as f:
+        f.write(data)
+        f.flush()
+
+
 class OwnCloudBackupAgent(BackupAgent):
     """Backup agent storing backups in ownCloud via WebDAV."""
 
@@ -61,21 +162,32 @@ class OwnCloudBackupAgent(BackupAgent):
         backup: AgentBackup,
         **kwargs: Any,
     ) -> None:
-        """Upload a backup + metadata sidecar."""
+        """Upload a backup + metadata sidecar (spooled for non-chunked PUT)."""
+        temp_path: str | None = None
         try:
             tar_name = _make_tar_name(backup.backup_id)
             meta_name = _make_meta_name(backup.backup_id)
 
-            # 1) Upload tar stream
+            # 1) Spool tar stream to temp file
             stream = await open_stream()
-            await self._client.put_stream(tar_name, stream)
+            temp_path, size = await _spool_stream_to_tempfile(stream)
 
-            # 2) Upload metadata JSON (small)
-            meta_bytes = json.dumps(backup.to_dict(), ensure_ascii=False).encode("utf-8")
+            # 2) Upload tar file with Content-Length
+            await self._client.put_file(tar_name, temp_path, size)
+
+            # 3) Upload normalized metadata JSON
+            meta_dict = _agentbackup_to_dict(backup)
+            meta_bytes = json.dumps(meta_dict, ensure_ascii=False).encode("utf-8")
             await self._client.put_bytes(meta_name, meta_bytes)
 
         except Exception as err:  # noqa: BLE001
             raise BackupAgentError(f"Upload to ownCloud failed: {err}") from err
+        finally:
+            if temp_path:
+                try:
+                    os.remove(temp_path)
+                except OSError:
+                    pass
 
     async def async_list_backups(self, **kwargs: Any) -> list[AgentBackup]:
         """List backups by reading metadata sidecars; fallback to tar stat if missing."""
@@ -87,7 +199,6 @@ class OwnCloudBackupAgent(BackupAgent):
 
             backups: list[AgentBackup] = []
 
-            # 1) Load metadata sidecars (limited concurrency)
             sem = asyncio.Semaphore(5)
 
             async def fetch_meta(meta_name: str) -> None:
@@ -101,7 +212,6 @@ class OwnCloudBackupAgent(BackupAgent):
 
             await asyncio.gather(*(fetch_meta(m) for m in meta_files))
 
-            # 2) Fallback: tar without meta -> synthesize minimal AgentBackup
             known_ids = {b.backup_id for b in backups}
             for tar_name in tar_files:
                 backup_id = tar_name.removeprefix(TAR_PREFIX).removesuffix(TAR_SUFFIX)
@@ -109,13 +219,21 @@ class OwnCloudBackupAgent(BackupAgent):
                     continue
 
                 info = await self._client.stat(tar_name)
-                d = {
-                    "backup_id": backup_id,
-                    "name": f"ownCloud backup ({backup_id})",
-                    "date": info.get("modified_iso", ""),
-                    "size": info.get("size", 0),
-                    "protected": False,
-                }
+                d = _normalize_backup_dict(
+                    {
+                        "backup_id": backup_id,
+                        "name": f"ownCloud backup ({backup_id})",
+                        "date": info.get("modified_iso", ""),
+                        "size": info.get("size", 0),
+                        # conservative defaults:
+                        "database_included": True,
+                        "homeassistant_included": True,
+                        "addons_included": False,
+                        "folders_included": False,
+                        "addons": [],
+                        "folders": [],
+                    }
+                )
                 backups.append(_agentbackup_from_dict(d))
 
             backups.sort(key=lambda b: str(b.date), reverse=True)
@@ -129,7 +247,6 @@ class OwnCloudBackupAgent(BackupAgent):
         meta_name = _make_meta_name(backup_id)
         tar_name = _make_tar_name(backup_id)
 
-        # 1) Try meta
         try:
             raw = await self._client.get_bytes(meta_name)
             d = json.loads(raw.decode("utf-8"))
@@ -139,16 +256,22 @@ class OwnCloudBackupAgent(BackupAgent):
         except Exception as err:  # noqa: BLE001
             raise BackupAgentError(f"Get backup metadata failed: {err}") from err
 
-        # 2) Fallback to tar stat
         try:
             info = await self._client.stat(tar_name)
-            d = {
-                "backup_id": backup_id,
-                "name": f"ownCloud backup ({backup_id})",
-                "date": info.get("modified_iso", ""),
-                "size": info.get("size", 0),
-                "protected": False,
-            }
+            d = _normalize_backup_dict(
+                {
+                    "backup_id": backup_id,
+                    "name": f"ownCloud backup ({backup_id})",
+                    "date": info.get("modified_iso", ""),
+                    "size": info.get("size", 0),
+                    "database_included": True,
+                    "homeassistant_included": True,
+                    "addons_included": False,
+                    "folders_included": False,
+                    "addons": [],
+                    "folders": [],
+                }
+            )
             return _agentbackup_from_dict(d)
         except FileNotFoundError as err:
             raise BackupNotFound(f"Backup not found: {backup_id}") from err

custom_components/owncloud_backup/const.py

@@ -14,3 +14,6 @@ DATA_BACKUP_AGENT_LISTENERS = "backup_agent_listeners"
 TAR_PREFIX = "ha_backup_"
 TAR_SUFFIX = ".tar"
 META_SUFFIX = ".json"
+
+# Spooling to temp file to avoid chunked WebDAV uploads
+SPOOL_FLUSH_BYTES = 1024 * 1024  # 1 MiB

custom_components/owncloud_backup/manifest.json

@@ -1,10 +1,10 @@
 {
   "domain": "owncloud_backup",
   "name": "ownCloud Backup (WebDAV)",
-  "version": "0.1.0-alpha",
-  "documentation": "https://github.com/your-org/owncloud-backup-ha",
-  "issue_tracker": "https://github.com/your-org/owncloud-backup-ha/issues",
-  "codeowners": [],
+  "version": "0.2.0",
+  "documentation": "https://github.com/bahmcloud/owncloud-backup-ha/",
+  "issue_tracker": "https://github.com/bahmcloud/owncloud-backup-ha/issues",
+  "codeowners": ["@bahmcloud"],
   "config_flow": true,
   "integration_type": "service",
   "iot_class": "cloud_push",

custom_components/owncloud_backup/webdav_client.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 
 import base64
 import logging
+import os
 import xml.etree.ElementTree as ET
 from collections.abc import AsyncIterator
 from datetime import datetime, timezone
@@ -9,6 +10,7 @@ from email.utils import parsedate_to_datetime
 from typing import Final
 from urllib.parse import quote, urljoin
 
+import aiohttp
 from aiohttp import ClientResponseError, ClientSession
 from yarl import URL
 
@@ -46,6 +48,11 @@ class WebDavClient:
         ]
         self._cached_root: str | None = None
 
+        # Non-restrictive client timeouts for potentially long WebDAV operations
+        self._timeout_long = aiohttp.ClientTimeout(
+            total=None, connect=60, sock_connect=60, sock_read=None
+        )
+
     def _auth_header(self) -> str:
         token = base64.b64encode(f"{self._username}:{self._password}".encode("utf-8")).decode("ascii")
         return f"Basic {token}"
@@ -82,6 +89,7 @@ class WebDavClient:
                         b'<d:propfind xmlns:d="DAV:"><d:prop><d:resourcetype/></d:prop></d:propfind>'
                     ),
                     raise_for_status=True,
+                    timeout=self._timeout_long,
                 ):
                     self._cached_root = root
                     return root
@@ -106,6 +114,7 @@ class WebDavClient:
                 base_folder,
                 headers=self._headers({"Depth": "0"}),
                 raise_for_status=True,
+                timeout=self._timeout_long,
             ):
                 return
         except ClientResponseError as err:
@@ -127,7 +136,11 @@ class WebDavClient:
         # exists?
         try:
             async with self._session.request(
-                "PROPFIND", url, headers=self._headers({"Depth": "0"}), raise_for_status=True
+                "PROPFIND",
+                url,
+                headers=self._headers({"Depth": "0"}),
+                raise_for_status=True,
+                timeout=self._timeout_long,
             ):
                 return
         except ClientResponseError as err:
@@ -135,7 +148,9 @@ class WebDavClient:
                 raise
 
         # create
-        async with self._session.request("MKCOL", url, headers=self._headers()) as resp:
+        async with self._session.request(
+            "MKCOL", url, headers=self._headers(), timeout=self._timeout_long
+        ) as resp:
             if resp.status in (201, 405):
                 return
             text = await resp.text()
@@ -154,6 +169,7 @@ class WebDavClient:
                 b'<d:propfind xmlns:d="DAV:"><d:prop><d:displayname/></d:prop></d:propfind>'
             ),
             raise_for_status=True,
+            timeout=self._timeout_long,
         ) as resp:
             body = await resp.text()
 
@@ -195,24 +211,44 @@ class WebDavClient:
     async def put_bytes(self, name: str, data: bytes) -> None:
         folder = await self._base_folder_url()
         url = self._file_url(folder, name)
-        async with self._session.put(url, data=data, headers=self._headers(), raise_for_status=True):
+        async with self._session.put(
+            url,
+            data=data,
+            headers=self._headers({"Content-Length": str(len(data))}),
+            raise_for_status=True,
+            timeout=self._timeout_long,
+        ):
             return
 
-    async def put_stream(self, name: str, stream: AsyncIterator[bytes]) -> None:
+    async def put_file(self, name: str, path: str, size: int) -> None:
+        """Upload a local file with an explicit Content-Length (non-chunked)."""
         folder = await self._base_folder_url()
         url = self._file_url(folder, name)
 
-        async def gen():
-            async for chunk in stream:
-                yield chunk
+        # Ensure correct size if caller passes 0/unknown
+        if size <= 0:
+            try:
+                size = os.path.getsize(path)
+            except OSError:
+                size = 0
 
-        async with self._session.put(url, data=gen(), headers=self._headers(), raise_for_status=True):
-            return
+        headers = {"Content-Length": str(size)} if size > 0 else {}
+
+        # aiohttp will stream file content; with Content-Length set, proxies are usually happier.
+        with open(path, "rb") as f:
+            async with self._session.put(
+                url,
+                data=f,
+                headers=self._headers(headers),
+                raise_for_status=True,
+                timeout=self._timeout_long,
+            ):
+                return
 
     async def get_bytes(self, name: str) -> bytes:
         folder = await self._base_folder_url()
         url = self._file_url(folder, name)
-        async with self._session.get(url, headers=self._headers()) as resp:
+        async with self._session.get(url, headers=self._headers(), timeout=self._timeout_long) as resp:
             if resp.status == 404:
                 raise FileNotFoundError(name)
             resp.raise_for_status()
@@ -221,7 +257,7 @@ class WebDavClient:
     async def get_stream(self, name: str) -> AsyncIterator[bytes]:
         folder = await self._base_folder_url()
         url = self._file_url(folder, name)
-        resp = await self._session.get(url, headers=self._headers())
+        resp = await self._session.get(url, headers=self._headers(), timeout=self._timeout_long)
         if resp.status == 404:
             await resp.release()
             raise FileNotFoundError(name)
@@ -239,7 +275,7 @@ class WebDavClient:
     async def delete(self, name: str) -> None:
         folder = await self._base_folder_url()
         url = self._file_url(folder, name)
-        async with self._session.delete(url, headers=self._headers()) as resp:
+        async with self._session.delete(url, headers=self._headers(), timeout=self._timeout_long) as resp:
             if resp.status == 404:
                 raise FileNotFoundError(name)
             if resp.status in (200, 202, 204):
@@ -267,6 +303,7 @@ class WebDavClient:
             url,
             headers=self._headers({"Depth": "0", "Content-Type": "application/xml; charset=utf-8"}),
             data=body,
+            timeout=self._timeout_long,
         ) as resp:
             if resp.status == 404:
                 raise FileNotFoundError(name)