From ec2a02d581b22678eaa41ffae0fa7dce2b6be92d Mon Sep 17 00:00:00 2001 From: ledadmin Date: Thu, 3 Sep 2026 11:59:21 -0400 Subject: [PATCH] Initial DataLoom project scaffold --- .gitignore | 20 ++++++++++ README.md | 29 ++++++++++++++ docs/design.md | 67 +++++++++++++++++++++++++++++++++ docs/roadmap.md | 12 ++++++ pyproject.toml | 17 +++++++++ src/dataloom/__init__.py | 3 ++ src/dataloom/app.py | 6 +++ tests/fixtures/README.md | 3 ++ tests/fixtures/bitwarden-a.json | 24 ++++++++++++ tests/fixtures/bitwarden-b.json | 24 ++++++++++++ 10 files changed, 205 insertions(+) create mode 100644 .gitignore create mode 100644 README.md create mode 100644 docs/design.md create mode 100644 docs/roadmap.md create mode 100644 pyproject.toml create mode 100644 src/dataloom/__init__.py create mode 100644 src/dataloom/app.py create mode 100644 tests/fixtures/README.md create mode 100644 tests/fixtures/bitwarden-a.json create mode 100644 tests/fixtures/bitwarden-b.json diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4e45676 --- /dev/null +++ b/.gitignore @@ -0,0 +1,20 @@ +__pycache__/ +*.py[cod] +*.egg-info/ +.venv/ +.pytest_cache/ +.mypy_cache/ +.ruff_cache/ +dist/ +build/ + +# Sensitive vault exports, logs, and local runtime state +*.json +*.csv +*.log +*.txt +!README.md +!docs/*.md +!tests/fixtures/*.json +!tests/fixtures/*.csv +!tests/fixtures/*.txt diff --git a/README.md b/README.md new file mode 100644 index 0000000..4b536ff --- /dev/null +++ b/README.md @@ -0,0 +1,29 @@ +# DataLoom + +Offline desktop application for reviewing and merging Bitwarden exports. + +## Status + +Initial project scaffold and design documentation. Implementation has not started. + +## Goals + +- Import two or more same-format, unencrypted Bitwarden exports. +- Support JSON and CSV input, preserving all representable Bitwarden data. +- Automatically remove only exact duplicates. +- Keep uncertain and conflicting records for review. +- Produce one merged output in the same format as the inputs. +- Provide a comprehensive on-screen and plain-text audit log. +- Operate offline with no uploads or external service dependencies. + +## Security note + +Bitwarden exports and the requested full audit log contain secrets. DataLoom must display a prominent warning before import and clearly identify all generated files and their locations. Real exports must never be committed to source control. + +## Planned technology + +Python and PySide6/Qt for a desktop application. The initial implementation will use synthetic fixtures only. + +## Development + +The detailed design is in [`docs/design.md`](docs/design.md). See [`docs/roadmap.md`](docs/roadmap.md) for the planned milestones. diff --git a/docs/design.md b/docs/design.md new file mode 100644 index 0000000..d1beefd --- /dev/null +++ b/docs/design.md @@ -0,0 +1,67 @@ +# DataLoom design + +## Application + +DataLoom is an offline PySide6/Qt desktop application. It does not upload files, call cloud services, or require a network connection during normal operation. + +## Import rules + +- Accept two or more files in one batch. +- Every file in a batch must be Bitwarden JSON or every file must be Bitwarden CSV. +- Reject mixed-format batches and encrypted/protected exports. +- Validate the expected Bitwarden structure before processing. +- Preserve supported item types and unknown JSON fields where possible. +- Keep source filename, input index, record index, and Bitwarden item ID as internal provenance metadata for review and logging; do not add it to exported vault records. + +## Duplicate policy + +Only exact duplicates are removed automatically. All uncertain or conflicting records remain in the merged collection and are shown in Duplicate Review. + +### Exact duplicate + +Two records are exact duplicates when they have the same item type and equivalent normalized content, including the relevant login username, password, URI data, folder, notes, custom fields, and other representable fields. Comparisons are case-insensitive where appropriate. + +### Likely duplicate + +Same item type, normalized URL identity, and username, but one or more values differ. These records remain separate. + +### Possible duplicate + +Same item type and normalized URL identity, but different usernames or other identity fields. These records remain separate. + +Non-login records use type-appropriate stable fields and full normalized content. Folder membership is retained but does not alone determine duplication. + +## URL normalization + +Normalize URLs conservatively and case-insensitively by parsing the URL, lowercasing the scheme/host, removing default ports, removing trailing slashes, and applying the configured `www.` equivalence. Query strings and paths remain significant unless Bitwarden-specific behavior establishes otherwise. The original URI is always preserved for export. + +When normalization is ambiguous, retain both records and flag them for review. + +## Newest record preference + +When duplicate records conflict, use Bitwarden `revisionDate` when available and valid. The newest record is preferred only where the timestamp is reliable. CSV commonly lacks sufficient revision metadata; in that case DataLoom keeps a deterministic first record in the merged output, retains all conflicting records in Duplicate Review, and logs that newest selection was unavailable. + +## Review interface + +1. **Import Summary** — files, format, counts, validation warnings, and output locations. +2. **Merged Entries** — records included in the output, with provenance and classification. +3. **Duplicate Review** — grouped likely/possible duplicates and exact-duplicate removal decisions, with side-by-side differences. +4. **Audit Log** — live in-app view and persistent plain-text log. + +The workflow is automated; there is no general record editor. Review is read-only in the first version. + +## Audit logging + +The requested log records every file and record viewed, fields inspected, comparisons, classifications, selection decisions, warnings, and exports, including secret values. Because this log is equivalent in sensitivity to a vault, the import warning must explicitly state that it contains secrets. The user is responsible for retaining and deleting the log. DataLoom must never log secrets from its own synthetic fixtures into repository files. + +## Export + +- JSON input produces JSON output. +- CSV input produces CSV output. +- JSON is the lossless preferred format. +- Before CSV export, report any fields that CSV cannot represent completely (for example, nested/custom structures, passkey data, or attachment metadata). +- Never silently discard unsupported data. + +## Data lifecycle + +DataLoom does not automatically delete input, output, or log files. After processing it displays the exact locations of all generated files. Real exports and generated vault/log files are ignored by Git. diff --git a/docs/roadmap.md b/docs/roadmap.md new file mode 100644 index 0000000..f79b0bf --- /dev/null +++ b/docs/roadmap.md @@ -0,0 +1,12 @@ +# Roadmap + +1. Define internal record model and synthetic JSON/CSV fixtures. +2. Implement strict same-format Bitwarden import and validation. +3. Implement loss-aware field preservation and URL normalization. +4. Implement exact/likely/possible duplicate classification and newest-record preference. +5. Implement same-format merged export with CSV loss warnings. +6. Build PySide6 desktop workflow and read-only review screens. +7. Add comprehensive plain-text and in-app audit logging. +8. Add packaging documentation and desktop smoke tests. + +Real Bitwarden exports are not required for development or tests. diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..733f40a --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,17 @@ +[build-system] +requires = ["setuptools>=68"] +build-backend = "setuptools.build_meta" + +[project] +name = "dataloom" +version = "0.1.0" +description = "Offline Bitwarden export review and merge desktop application" +readme = "README.md" +requires-python = ">=3.11" +dependencies = ["PySide6>=6.6"] + +[project.scripts] +dataloom = "dataloom.app:main" + +[tool.setuptools.packages.find] +where = ["src"] diff --git a/src/dataloom/__init__.py b/src/dataloom/__init__.py new file mode 100644 index 0000000..9e187c2 --- /dev/null +++ b/src/dataloom/__init__.py @@ -0,0 +1,3 @@ +"""DataLoom offline Bitwarden export review application.""" + +__version__ = "0.1.0" diff --git a/src/dataloom/app.py b/src/dataloom/app.py new file mode 100644 index 0000000..09ebd5d --- /dev/null +++ b/src/dataloom/app.py @@ -0,0 +1,6 @@ +"""Application entry point placeholder for the DataLoom desktop UI.""" + + +def main() -> int: + """Start the desktop application once the UI is implemented.""" + raise NotImplementedError("DataLoom UI is not implemented yet") diff --git a/tests/fixtures/README.md b/tests/fixtures/README.md new file mode 100644 index 0000000..26452aa --- /dev/null +++ b/tests/fixtures/README.md @@ -0,0 +1,3 @@ +# Synthetic fixtures + +Fixtures in this directory must contain fake credentials only. Never place a real Bitwarden export here. diff --git a/tests/fixtures/bitwarden-a.json b/tests/fixtures/bitwarden-a.json new file mode 100644 index 0000000..6c97878 --- /dev/null +++ b/tests/fixtures/bitwarden-a.json @@ -0,0 +1,24 @@ +{ + "encrypted": false, + "folders": [], + "items": [ + { + "id": "synthetic-a-1", + "organizationId": null, + "folderId": null, + "type": 1, + "reprompt": 0, + "name": "Example Account", + "notes": "synthetic fixture only", + "favorite": false, + "login": { + "username": "fake-user", + "password": "not-a-real-password", + "uris": [{"match": null, "uri": "https://example.com"}], + "totp": null + }, + "revisionDate": "2026-01-01T00:00:00.000Z", + "fields": [] + } + ] +} diff --git a/tests/fixtures/bitwarden-b.json b/tests/fixtures/bitwarden-b.json new file mode 100644 index 0000000..61dd7b5 --- /dev/null +++ b/tests/fixtures/bitwarden-b.json @@ -0,0 +1,24 @@ +{ + "encrypted": false, + "folders": [], + "items": [ + { + "id": "synthetic-b-1", + "organizationId": null, + "folderId": null, + "type": 1, + "reprompt": 0, + "name": "Example Account Copy", + "notes": "synthetic fixture only", + "favorite": false, + "login": { + "username": "fake-user", + "password": "not-a-real-password", + "uris": [{"match": null, "uri": "https://www.example.com/"}], + "totp": null + }, + "revisionDate": "2026-02-01T00:00:00.000Z", + "fields": [] + } + ] +}