class RemarkableUploadManager_v1
Manages uploads to reMarkable cloud
/tf/active/vicechatdev/e-ink-llm/cloudtest/upload_manager_old.py
32 - 665
moderate
Purpose
Manages uploads to reMarkable cloud
Source Code
class RemarkableUploadManager:
"""Manages uploads to reMarkable cloud"""
def __init__(self, session: requests.Session, replica_database_path: str):
self.session = session
self.base_url = "https://eu.tectonic.remarkable.com"
# Load replica database
self.database_path = Path(replica_database_path)
self.database = self._load_database()
# Track uploads
self.upload_queue: List[Dict[str, Any]] = []
self.uploaded_hashes: Dict[str, str] = {} # hash -> upload_status
self._current_document_uuid: Optional[str] = None # UUID for consistent rm-filename headers
def _clear_document_context(self):
"""Clear the current document UUID context for new uploads"""
self._current_document_uuid = None
def _load_database(self) -> Dict[str, Any]:
"""Load the replica database"""
if not self.database_path.exists():
raise FileNotFoundError(f"Database not found: {self.database_path}")
with open(self.database_path, 'r', encoding='utf-8') as f:
return json.load(f)
def _save_database(self):
"""Save the updated database"""
with open(self.database_path, 'w', encoding='utf-8') as f:
json.dump(self.database, f, indent=2, ensure_ascii=False)
def _compute_hash(self, content: bytes) -> str:
"""Compute SHA256 hash of content"""
return hashlib.sha256(content).hexdigest()
def _compute_crc32c_header(self, content: bytes) -> str:
"""Compute CRC32C checksum and return as x-goog-hash header value"""
try:
# Use proper crc32c library if available
if HAS_CRC32C:
checksum = crc32c.crc32c(content)
else:
# Fallback to standard CRC32 (not ideal but better than nothing)
checksum = zlib.crc32(content) & 0xffffffff
# Convert to bytes and base64 encode
checksum_bytes = checksum.to_bytes(4, byteorder='big')
checksum_b64 = base64.b64encode(checksum_bytes).decode('ascii')
return f"crc32c={checksum_b64}"
except Exception as e:
print(f"⚠️ Warning: Failed to compute CRC32C checksum: {e}")
# Return empty string to skip the header if computation fails
return ""
def _generate_timestamp(self) -> str:
"""Generate reMarkable timestamp"""
return str(int(time.time() * 1000))
def _generate_generation(self) -> int:
"""Generate reMarkable generation number"""
return int(time.time() * 1000000)
def upload_raw_content(self, content: bytes, content_hash: str = None, filename: str = None,
content_type: str = "application/octet-stream", system_filename: str = None) -> Optional[str]:
"""Upload raw content and return its hash"""
if content_hash is None:
content_hash = self._compute_hash(content)
# Check if already uploaded
if content_hash in self.uploaded_hashes:
print(f"✅ Content already uploaded: {content_hash[:16]}...")
return content_hash
try:
url = f"{self.base_url}/sync/v3/files/{content_hash}"
# Prepare headers like the reMarkable app
headers = {
'Content-Type': content_type,
'rm-batch-number': '1',
'rm-sync-id': str(uuid.uuid4()),
'User-Agent': 'desktop/3.20.0.922 (macos 15.4)',
'Accept-Encoding': 'gzip, deflate',
'Accept-Language': 'en-BE,*',
'Connection': 'Keep-Alive'
}
# Add rm-filename header - REQUIRED for all PUT requests
# Handle different patterns: UUID-based files vs system files
if system_filename:
# System files like "roothash", "root.docSchema" (no UUID)
rm_filename = system_filename
print(f"🏷️ rm-filename (system): {rm_filename}")
elif filename:
# Document files with UUID pattern
if hasattr(self, '_current_document_uuid') and self._current_document_uuid:
doc_uuid = self._current_document_uuid
else:
# Generate and store new UUID for this document
doc_uuid = str(uuid.uuid4())
self._current_document_uuid = doc_uuid
print(f"📊 Generated new document UUID: {doc_uuid}")
# Use the filename as provided or construct UUID.extension format
if '.' in filename and len(filename.split('.')[0]) == 36: # Already UUID.extension
rm_filename = filename
else:
# Determine extension and construct UUID.extension
if content_type == 'application/pdf' or filename.lower().endswith('.pdf'):
rm_filename = f"{doc_uuid}.pdf"
elif 'metadata' in filename.lower():
rm_filename = f"{doc_uuid}.metadata"
elif filename.lower().endswith('.content'):
rm_filename = f"{doc_uuid}.content"
elif filename.lower().endswith('.rm'):
# Page data keeps original filename for .rm files
rm_filename = filename
elif filename.lower().endswith('.docschema') or 'docschema' in filename.lower():
rm_filename = f"{doc_uuid}.docSchema"
elif filename.lower().endswith('.pagedata'):
rm_filename = f"{doc_uuid}.pagedata"
else:
# Default construction
rm_filename = f"{doc_uuid}.{filename}"
print(f"🏷️ rm-filename (document): {rm_filename}")
else:
# Fallback - generate basic filename
if hasattr(self, '_current_document_uuid') and self._current_document_uuid:
doc_uuid = self._current_document_uuid
else:
doc_uuid = str(uuid.uuid4())
self._current_document_uuid = doc_uuid
if content_type == 'application/pdf':
rm_filename = f"{doc_uuid}.pdf"
elif content_type == 'application/octet-stream':
rm_filename = f"{doc_uuid}.metadata"
else:
rm_filename = f"{doc_uuid}.content"
print(f"🏷️ rm-filename (fallback): {rm_filename}")
headers['rm-filename'] = rm_filename
# Add CRC32C checksum (this is the missing piece!)
crc32c_header = self._compute_crc32c_header(content)
if crc32c_header:
headers['x-goog-hash'] = crc32c_header
print(f"🔍 Debug: Upload headers for {content_hash[:16]}...")
for key, value in headers.items():
print(f" {key}: {value}")
# Make the PUT request
response = self.session.put(url, data=content, headers=headers)
print(f"🔍 Debug: Response status: {response.status_code}")
print(f"🔍 Debug: Response text: {response.text}")
response.raise_for_status()
self.uploaded_hashes[content_hash] = "uploaded"
print(f"✅ Uploaded content: {content_hash[:16]}... ({len(content)} bytes)")
return content_hash
except Exception as e:
print(f"❌ Failed to upload content {content_hash[:16]}...: {e}")
if hasattr(e, 'response') and e.response is not None:
print(f" Response: {e.response.text}")
return None
def upload_system_file(self, content: bytes, system_filename: str, content_type: str = "application/octet-stream") -> Optional[str]:
"""Upload system files like roothash, root.docSchema with fixed filenames"""
print(f"📁 Uploading system file: {system_filename}")
return self.upload_raw_content(content, system_filename=system_filename, content_type=content_type)
def upload_document_file(self, content: bytes, filename: str, content_type: str = "application/octet-stream") -> Optional[str]:
"""Upload document files with UUID.extension pattern"""
print(f"📄 Uploading document file: {filename}")
return self.upload_raw_content(content, filename=filename, content_type=content_type)
def create_metadata_json(self, name: str, parent_uuid: str = "", document_type: str = "DocumentType") -> Tuple[bytes, str]:
"""Create metadata JSON for a document"""
timestamp = self._generate_timestamp()
metadata = {
"createdTime": timestamp,
"lastModified": timestamp,
"lastOpened": timestamp,
"lastOpenedPage": 0,
"new": False,
"parent": parent_uuid,
"pinned": False,
"source": "",
"type": document_type,
"visibleName": name
}
content = json.dumps(metadata, indent=4).encode('utf-8')
content_hash = self._compute_hash(content)
return content, content_hash
def create_content_json(self, pages: List[str], template: str = "Blank") -> Tuple[bytes, str]:
"""Create content JSON for a notebook with pages"""
timestamp_base = f"2:{len(pages)}"
# Create pages structure
pages_list = []
for i, page_id in enumerate(pages):
pages_list.append({
"id": page_id,
"idx": {
"timestamp": f"2:{i+2}",
"value": chr(ord('a') + i) if i < 26 else f"page_{i}"
},
"template": {
"timestamp": "2:1",
"value": template
}
})
content_data = {
"cPages": {
"lastOpened": {
"timestamp": "2:1",
"value": pages[0] if pages else ""
},
"original": {
"timestamp": "0:0",
"value": -1
},
"pages": pages_list
},
"extraMetadata": {},
"fileType": "notebook",
"fontName": "",
"lineHeight": -1,
"margins": 180,
"pageCount": len(pages),
"textScale": 1,
"transform": {}
}
content = json.dumps(content_data, indent=4).encode('utf-8')
content_hash = self._compute_hash(content)
return content, content_hash
def create_directory_listing(self, child_objects: List[Dict], data_components: List[Dict]) -> Tuple[bytes, str]:
"""Create directory listing content"""
lines = [str(len(child_objects) + len(data_components))]
# Add child objects (folders/documents)
for obj in child_objects:
line = f"{obj['hash']}:80000000:{obj['uuid']}:{obj['type']}:{obj['size']}"
lines.append(line)
# Add data components (.content, .metadata, .rm files, etc.)
for comp in data_components:
line = f"{comp['hash']}:0:{comp['component']}:0:{comp['size']}"
lines.append(line)
content = '\n'.join(lines).encode('utf-8')
content_hash = self._compute_hash(content)
return content, content_hash
def update_root_hash(self, new_root_hash: str) -> bool:
"""Update the root hash in the cloud"""
try:
generation = self._generate_generation()
root_data = {
"broadcast": True,
"generation": generation,
"hash": new_root_hash
}
url = f"{self.base_url}/sync/v3/root"
response = self.session.put(url, json=root_data)
response.raise_for_status()
print(f"✅ Updated root hash: {new_root_hash}")
return True
except Exception as e:
print(f"❌ Failed to update root hash: {e}")
return False
def edit_document_metadata(self, document_uuid: str, new_name: str = None, new_parent: str = None) -> bool:
"""Edit an existing document's metadata"""
try:
# Find the document in database
if document_uuid not in self.database['nodes']:
raise ValueError(f"Document {document_uuid} not found in database")
node = self.database['nodes'][document_uuid]
print(f"📝 Editing document: {node['name']}")
# Get current metadata
current_metadata = node['metadata'].copy()
# Update metadata
if new_name:
current_metadata['visibleName'] = new_name
if new_parent is not None:
current_metadata['parent'] = new_parent
current_metadata['lastModified'] = self._generate_timestamp()
# Create new metadata content
metadata_content = json.dumps(current_metadata, indent=4).encode('utf-8')
metadata_hash = self._compute_hash(metadata_content)
# Upload metadata
self.upload_raw_content(metadata_content, metadata_hash)
# Update component hashes
old_metadata_hash = node['component_hashes']['metadata']
node['component_hashes']['metadata'] = metadata_hash
# Get parent node to update its directory listing
parent_uuid = current_metadata.get('parent', '')
if parent_uuid and parent_uuid in self.database['nodes']:
parent_node = self.database['nodes'][parent_uuid]
# Rebuild parent's directory listing
child_objects = []
data_components = []
# Find all children of this parent
for uuid, child_node in self.database['nodes'].items():
if child_node.get('parent_uuid') == parent_uuid:
if child_node['node_type'] == 'folder':
type_val = '1'
else:
type_val = '3'
child_objects.append({
'hash': child_node['hash'],
'uuid': uuid,
'type': type_val,
'size': len(str(child_node).encode('utf-8')) # Approximate
})
# Add metadata components for this updated document
comp_hashes = node['component_hashes']
for comp_type, comp_hash in comp_hashes.items():
if comp_hash:
if comp_type == 'rm_files':
for i, rm_hash in enumerate(comp_hash):
data_components.append({
'hash': rm_hash,
'component': f"{document_uuid}/{uuid.uuid4()}.rm",
'size': 14661 # Typical RM file size
})
else:
data_components.append({
'hash': comp_hash,
'component': f"{document_uuid}.{comp_type}",
'size': len(metadata_content) if comp_type == 'metadata' else 2209
})
# Create and upload new directory listing
dir_content, dir_hash = self.create_directory_listing(child_objects, data_components)
self.upload_raw_content(dir_content, dir_hash)
# Update parent node hash
parent_node['hash'] = dir_hash
self.database['hash_registry'][dir_hash] = {
'uuid': parent_uuid,
'type': 'node',
'last_seen': datetime.now().isoformat()
}
# Update root if parent is root
if not parent_node.get('parent_uuid'):
self.update_root_hash(dir_hash)
# Update database
node['metadata'] = current_metadata
node['last_modified'] = current_metadata['lastModified']
node['sync_status'] = 'updated'
node['last_synced'] = datetime.now().isoformat()
# Update hash registry
self.database['hash_registry'][metadata_hash] = {
'uuid': document_uuid,
'type': 'metadata',
'last_seen': datetime.now().isoformat()
}
self._save_database()
print(f"✅ Successfully updated document metadata")
return True
except Exception as e:
print(f"❌ Failed to edit document metadata: {e}")
return False
def upload_pdf_document(self, pdf_path: str, name: str, parent_uuid: str = "") -> bool:
"""Upload a new PDF document to reMarkable following the correct sequence from app logs"""
try:
# Clear any previous document context
self._clear_document_context()
pdf_file = Path(pdf_path)
if not pdf_file.exists():
raise FileNotFoundError(f"PDF file not found: {pdf_path}")
print(f"📄 Uploading PDF: {name}")
# Generate UUID for new document and set it for consistent rm-filename headers
document_uuid = str(uuid.uuid4())
self._current_document_uuid = document_uuid
print(f"📊 Document UUID: {document_uuid}")
# Read PDF content
with open(pdf_file, 'rb') as f:
pdf_content = f.read()
# FOLLOW APP LOGS UPLOAD ORDER:
# 1. Content (if any) - for PDFs this might be empty or minimal
# 2. Page data (.rm files) - not needed for PDF
# 3. Metadata
# 4. PDF content
print("📝 Step 1: Creating and uploading content...")
# Create minimal content for PDF (empty content structure)
content_data, content_hash = self.create_content_json([], "PDF")
self.upload_raw_content(
content=content_data,
content_type='application/octet-stream',
filename=f"{document_uuid}.content"
)
print("📝 Step 2: Creating and uploading metadata...")
# Create metadata
metadata_content, metadata_hash = self.create_metadata_json(name, parent_uuid)
self.upload_raw_content(
content=metadata_content,
content_type='application/octet-stream',
filename=f"{document_uuid}.metadata"
)
print("📝 Step 3: Uploading PDF content...")
# Upload PDF content LAST (as per app logs)
pdf_hash = self.upload_raw_content(
content=pdf_content,
content_type='application/pdf',
filename=f"{document_uuid}.pdf"
)
# Create document directory listing
data_components = [
{
'hash': metadata_hash,
'component': f"{document_uuid}.metadata",
'size': len(metadata_content)
},
{
'hash': pdf_hash,
'component': f"{document_uuid}.pdf",
'size': len(pdf_content)
}
]
doc_dir_content, doc_dir_hash = self.create_directory_listing([], data_components)
self.upload_raw_content(doc_dir_content, doc_dir_hash)
# Add to database
new_node = {
'uuid': document_uuid,
'hash': doc_dir_hash,
'name': name,
'node_type': 'document',
'parent_uuid': parent_uuid,
'local_path': f"content/{name}",
'extracted_files': [str(pdf_file)],
'component_hashes': {
'content': None,
'metadata': metadata_hash,
'pdf': pdf_hash,
'pagedata': None,
'rm_files': []
},
'metadata': json.loads(metadata_content.decode('utf-8')),
'last_modified': self._generate_timestamp(),
'version': 1,
'sync_status': 'uploaded',
'last_synced': datetime.now().isoformat()
}
self.database['nodes'][document_uuid] = new_node
# Update hash registry
for hash_val, info in [
(doc_dir_hash, {'uuid': document_uuid, 'type': 'node'}),
(metadata_hash, {'uuid': document_uuid, 'type': 'metadata'}),
(pdf_hash, {'uuid': document_uuid, 'type': 'pdf'})
]:
self.database['hash_registry'][hash_val] = {
**info,
'last_seen': datetime.now().isoformat()
}
# Update parent directory and root if needed
if parent_uuid and parent_uuid in self.database['nodes']:
# TODO: Update parent directory listing
pass
else:
# Document added to root - update root hash
self.update_root_hash(doc_dir_hash)
self._save_database()
print(f"✅ Successfully uploaded PDF document: {name}")
return True
except Exception as e:
print(f"❌ Failed to upload PDF document: {e}")
return False
def create_notebook(self, name: str, parent_uuid: str = "", template: str = "Blank") -> bool:
"""Create a new empty notebook"""
try:
# Clear any previous document context
self._clear_document_context()
print(f"📓 Creating notebook: {name}")
# Generate UUIDs and set current document UUID for consistent rm-filename headers
document_uuid = str(uuid.uuid4())
self._current_document_uuid = document_uuid
page_uuid = str(uuid.uuid4())
print(f"📊 Document UUID: {document_uuid}")
# Create empty .rm content for first page
rm_content = b'\x00' * 1000 # Minimal empty page content
rm_hash = self.upload_raw_content(
content=rm_content,
content_type='application/octet-stream',
filename=f"{page_uuid}.rm"
)
# Create content.json
content_data, content_hash = self.create_content_json([page_uuid], template)
self.upload_raw_content(
content=content_data,
content_type='application/octet-stream',
filename=f"{document_uuid}.content"
)
# Create metadata
metadata_content, metadata_hash = self.create_metadata_json(name, parent_uuid)
self.upload_raw_content(
content=metadata_content,
content_type='application/octet-stream',
filename=f"{document_uuid}.metadata"
)
# Create document directory listing
data_components = [
{
'hash': content_hash,
'component': f"{document_uuid}.content",
'size': len(content_data)
},
{
'hash': metadata_hash,
'component': f"{document_uuid}.metadata",
'size': len(metadata_content)
},
{
'hash': rm_hash,
'component': f"{document_uuid}/{page_uuid}.rm",
'size': len(rm_content)
}
]
doc_dir_content, doc_dir_hash = self.create_directory_listing([], data_components)
self.upload_raw_content(doc_dir_content, doc_dir_hash)
# Add to database
new_node = {
'uuid': document_uuid,
'hash': doc_dir_hash,
'name': name,
'node_type': 'document',
'parent_uuid': parent_uuid,
'local_path': f"content/{name}",
'extracted_files': [],
'component_hashes': {
'content': content_hash,
'metadata': metadata_hash,
'pdf': None,
'pagedata': None,
'rm_files': [rm_hash]
},
'metadata': json.loads(metadata_content.decode('utf-8')),
'last_modified': self._generate_timestamp(),
'version': 1,
'sync_status': 'created',
'last_synced': datetime.now().isoformat()
}
self.database['nodes'][document_uuid] = new_node
# Update hash registry
for hash_val, info in [
(doc_dir_hash, {'uuid': document_uuid, 'type': 'node'}),
(content_hash, {'uuid': document_uuid, 'type': 'content'}),
(metadata_hash, {'uuid': document_uuid, 'type': 'metadata'}),
(rm_hash, {'uuid': document_uuid, 'type': 'rm_0'})
]:
self.database['hash_registry'][hash_val] = {
**info,
'last_seen': datetime.now().isoformat()
}
# Update root hash (simplified for demo)
self.update_root_hash(doc_dir_hash)
self._save_database()
print(f"✅ Successfully created notebook: {name}")
return True
except Exception as e:
print(f"❌ Failed to create notebook: {e}")
return False
Parameters
| Name | Type | Default | Kind |
|---|---|---|---|
bases |
- | - |
Parameter Details
bases: Parameter of type
Return Value
Returns unspecified type
Class Interface
Methods
__init__(self, session, replica_database_path)
Purpose: Internal method: init
Parameters:
session: Type: requests.Sessionreplica_database_path: Type: str
Returns: None
_clear_document_context(self)
Purpose: Clear the current document UUID context for new uploads
Returns: None
_load_database(self) -> Dict[str, Any]
Purpose: Load the replica database
Returns: Returns Dict[str, Any]
_save_database(self)
Purpose: Save the updated database
Returns: None
_compute_hash(self, content) -> str
Purpose: Compute SHA256 hash of content
Parameters:
content: Type: bytes
Returns: Returns str
_compute_crc32c_header(self, content) -> str
Purpose: Compute CRC32C checksum and return as x-goog-hash header value
Parameters:
content: Type: bytes
Returns: Returns str
_generate_timestamp(self) -> str
Purpose: Generate reMarkable timestamp
Returns: Returns str
_generate_generation(self) -> int
Purpose: Generate reMarkable generation number
Returns: Returns int
upload_raw_content(self, content, content_hash, filename, content_type, system_filename) -> Optional[str]
Purpose: Upload raw content and return its hash
Parameters:
content: Type: bytescontent_hash: Type: strfilename: Type: strcontent_type: Type: strsystem_filename: Type: str
Returns: Returns Optional[str]
upload_system_file(self, content, system_filename, content_type) -> Optional[str]
Purpose: Upload system files like roothash, root.docSchema with fixed filenames
Parameters:
content: Type: bytessystem_filename: Type: strcontent_type: Type: str
Returns: Returns Optional[str]
upload_document_file(self, content, filename, content_type) -> Optional[str]
Purpose: Upload document files with UUID.extension pattern
Parameters:
content: Type: bytesfilename: Type: strcontent_type: Type: str
Returns: Returns Optional[str]
create_metadata_json(self, name, parent_uuid, document_type) -> Tuple[bytes, str]
Purpose: Create metadata JSON for a document
Parameters:
name: Type: strparent_uuid: Type: strdocument_type: Type: str
Returns: Returns Tuple[bytes, str]
create_content_json(self, pages, template) -> Tuple[bytes, str]
Purpose: Create content JSON for a notebook with pages
Parameters:
pages: Type: List[str]template: Type: str
Returns: Returns Tuple[bytes, str]
create_directory_listing(self, child_objects, data_components) -> Tuple[bytes, str]
Purpose: Create directory listing content
Parameters:
child_objects: Type: List[Dict]data_components: Type: List[Dict]
Returns: Returns Tuple[bytes, str]
update_root_hash(self, new_root_hash) -> bool
Purpose: Update the root hash in the cloud
Parameters:
new_root_hash: Type: str
Returns: Returns bool
edit_document_metadata(self, document_uuid, new_name, new_parent) -> bool
Purpose: Edit an existing document's metadata
Parameters:
document_uuid: Type: strnew_name: Type: strnew_parent: Type: str
Returns: Returns bool
upload_pdf_document(self, pdf_path, name, parent_uuid) -> bool
Purpose: Upload a new PDF document to reMarkable following the correct sequence from app logs
Parameters:
pdf_path: Type: strname: Type: strparent_uuid: Type: str
Returns: Returns bool
create_notebook(self, name, parent_uuid, template) -> bool
Purpose: Create a new empty notebook
Parameters:
name: Type: strparent_uuid: Type: strtemplate: Type: str
Returns: Returns bool
Required Imports
import os
import json
import hashlib
import requests
import uuid
Usage Example
# Example usage:
# result = RemarkableUploadManager(bases)
Similar Components
AI-powered semantic similarity - components with related functionality:
-
class RemarkableUploadManager 98.9% similar
-
class RemarkablePDFUploader_v1 78.1% similar
-
class RemarkableCloudManager 74.8% similar
-
class RemarkableUploadTests 70.8% similar
-
class RemarkablePDFUploader 70.4% similar