Compare commits

...

46 commits

Author SHA1 Message Date
93793bb580 docs(README): update data storage description from Nostr to rusqlite 2025-06-05 16:37:53 +03:00
1f4b63b1f8 docs(README): update local server URL to port 3004 in README 2025-06-05 16:19:07 +03:00
f35c7cd085 Merge pull request 'dev' () from dev into main
Reviewed-on: 
2025-04-01 13:19:02 +00:00
e893e14c26 Merge pull request 'Merge pull request 'dev' () from dev into main' () from main into dev
Reviewed-on: 
2025-04-01 13:17:26 +00:00
303b713d59 fix(properties): fetch prop values alongside prop ids and labels 2025-04-01 14:43:36 +03:00
46e9b4e48e docs(readme): document properties data flow 2025-04-01 00:02:35 +03:00
8c7946091f fix(labels): display correct labels in dropdown and when properties are added 2025-03-31 23:50:35 +03:00
9d21d9999f fix(labels): add set_is_fetching_labels signal to fetch_item_properties label to fix error 2025-03-31 18:07:36 +03:00
40bb35d6a8 feat(labels): add state to track when labels are being fetched(in progress) 2025-03-31 17:36:46 +03:00
ef7245b716 fix(labels): add property cache and refactor fetch_item_properties function to use it 2025-03-27 15:31:58 +03:00
5c3070bfc0 fix(labels): update set_property_labels signal in fetch_item_properties function to avoid repetitive label fetches 2025-03-26 14:48:58 +03:00
a9611a08e4 Merge branch 'dev' of forge.ftt.gmbh:ryanmwangi/Compware into dev 2025-03-25 15:52:40 +03:00
ebb1afd1af fix(properties): show property labels in custom property autocomplete dropdown 2025-03-25 15:51:49 +03:00
734e710d8f Merge pull request 'dev' () from dev into main
Reviewed-on: 
2025-03-24 20:57:41 +00:00
1f52901885 Merge branch 'main' into dev 2025-03-24 20:56:53 +00:00
f0356e9d0c Merge pull request 'Add CompareWare link to Readme' () from readme-link into main
Reviewed-on: 
2025-03-21 13:33:17 +00:00
69430fae8a fix(db): clean up property deletion to be scoped per url 2025-03-21 16:21:19 +03:00
fe98c56872 fix(db): check if the global_item_id column exists before trying to add it. 2025-03-21 01:45:16 +03:00
12f4043e83 feat(db): add global_item_id to track items accross urls 2025-03-21 01:32:32 +03:00
9a7a8e575c Add CompareWare link to Readme 2025-03-20 12:29:44 +00:00
3126d90f5a fix(auto search): retrace my steps to resolve ownership errors 2025-03-18 23:47:47 +03:00
8c1cab3615 Merge branch 'dev' of forge.ftt.gmbh:ryanmwangi/Compware into dev 2025-03-17 22:41:42 +03:00
85dce655e4 feat(auto-search): remove search button and have autosearch while typing 2025-03-17 22:41:23 +03:00
cdca9e7faa Merge branch 'main' of forge.ftt.gmbh:ryanmwangi/Compware into main 2025-03-15 16:53:56 +03:00
5465811781 docs(README): update readme with steps of docker deployment. 2025-03-15 16:53:18 +03:00
d6d0ab18ec Merge pull request 'dev' () from dev into main
Reviewed-on: 
2025-03-15 13:33:30 +00:00
d806c0c5dc Merge branch 'main' into dev 2025-03-15 13:33:10 +00:00
3ef759e5c2 buid(docker): add deckerfile and docker-compose.toml from dev 2025-03-15 16:31:23 +03:00
f87f88db2d build(docker): update Dockerfile to use Debian Bullseye and Rust 1.83.0.
-switch from Alpine to Debian for runtime
-add wasm-bindgen-cli
-update cargo-leptos version
2025-03-14 17:35:12 +03:00
947d244326 build(docker): dockerize compareware 2025-03-13 18:04:34 +03:00
db15e33ebd docs(README): update README to reflect changes in storage of name and description input 2025-03-12 14:41:21 +03:00
32e79ea609 fix(db): don't update item_order when updating an item 2025-03-12 02:22:00 +03:00
11e4935055 fix(db): retain name and description values upon refresh 2025-03-12 01:46:17 +03:00
7e5f3400ef feat(db): store name and description inputs in the properties table other than the items table 2025-03-11 23:27:52 +03:00
414e91a825 fix(db): preserve original item_order during updates 2025-03-10 20:48:17 +03:00
896de305cc fix(db): fix the issue of order preservation.
-Added CTE (Common Table Expression) to first get ordered items
-Changed result collection from HashMap to Vec to preserve order
-Process rows sequentially while maintaining item order
2025-03-10 17:39:39 +03:00
47c87159ae feat(db): add item_order to track the order that items are saved in 2025-03-10 17:00:38 +03:00
8ac1d77e06 test(db): add logging to tests 2025-03-06 15:09:28 +03:00
6c2442a82b fix(db): edit add_selected_property fn to Insert URL if it does not exists. 2025-03-06 14:32:52 +03:00
04457fef62 build(cargo.toml): remove unused dependencies 2025-03-05 19:15:08 +03:00
db29d1e05a test(db): test for selected properties addition, retrieval and duplicate prevention. 2025-03-05 16:50:26 +03:00
c96dacaaeb test(db): add url management test and use INSERT or IGNORE to handle url duplicates 2025-03-04 23:49:05 +03:00
aa9743fd2b test(db): test existence of selected properties table 2025-03-04 23:25:29 +03:00
88c6acd7e4 test(db): add test for property management. 2025-03-04 23:17:55 +03:00
505647b432 test(db): implement database layer tests.
-set up test structure
-implement item lifetime tests
2025-03-04 18:05:18 +03:00
1b99027dbf build(test): add testing dependencies to Cargo.toml 2025-03-03 16:18:07 +03:00
9 changed files with 976 additions and 423 deletions

5
.dockerignore Normal file
View file

@ -0,0 +1,5 @@
target/
**/*.rs.bk
node_modules/
Dockerfile
docker-compose.yml

3
.gitignore vendored
View file

@ -18,4 +18,5 @@ playwright/.cache/
.idea/
# Ignore database file
compareware.db
compareware.db
.qodo

436
Cargo.lock generated

File diff suppressed because it is too large Load diff

View file

@ -19,7 +19,7 @@ paste = "1.0"
wasm-bindgen = "=0.2.99"
rusqlite = { version = "0.27.0", optional = true}
serde = { version = "1.0", features = ["derive"] }
uuid = { version = "1.0", features = ["v4"] }
uuid = { version = "1.0", features = ["v4", "js"] }
web-sys = { version = "0.3", features = ["Event"] }
nostr-sdk = "0.37"
tokio = "1"

View file

@ -1,6 +1,7 @@
# CompareWare
# [CompareWare](https://compareware.org/)
CompareWare is an open-source platform for comparing tools (software, hardware, etc.) with structured, crowdsourced data. It combines **Leptos** for a modern, reactive frontend and **Nostr** for decentralized data storage.
CompareWare is an open-source platform for comparing tools (software, hardware, etc.) with structured, crowdsourced data.
It combines Rust's **Leptos** for a modern, reactive frontend and **rusqlite** for data storage.
## **Features**
- **Item Management**: Add, view, and manage items with metadata and key-value tags.
@ -25,23 +26,23 @@ CompareWare is an open-source platform for comparing tools (software, hardware,
```bash
cargo leptos serve
```
3. Open your browser at [http://localhost:3000](http://localhost:3000)
3. Open your browser at [localhost:3004](http://localhost:3004)
## **Database Schema**
### Key Concepts
- **PK (Primary Key)**: Unique identifier for table records (🔑)
- **FK (Foreign Key)**: Reference linking related tables (➡️)
- **Core (core properties)**: name and description.
### **Tables Overview**
### Tables Overview
### Core Tables
| Table | Columns (PK/FK) | Description | Example Data |
|-------|------------------|-------------|--------------|
| **urls** | `id` (PK), `url`, `created_at` | Stores comparison URLs | `1, "/laptops", 2024-03-01` |
| **items** | `id` (PK), `url_id` (FK), `name`, `description`, `wikidata_id` | Comparison items | `"item1", 1, "MacBook Pro", "16-inch", "Q214276"` |
| **properties** | `id` (PK), `name`, `global_usage_count` | Available properties | `25, "screen_size", 150` |
| **item_properties** | `item_id` (PK/FK), `property_id` (PK/FK), `value` | Item-specific values | `"item1", 25, "16 inches"` |
| **selected_properties** | `url_id` (PK/FK), `property_id` (PK/FK) | Active properties per URL | `1, 25` |
| **items** | `id` (PK), `url_id` (FK), `wikidata_id` | Comparison items | `"item1", 1, "Q214276"` |
| **properties** | `id` (PK), `name` | All available properties (including core) | `1.0, "name"`<br>`2.0, "description"`<br>`3.0, "screen_size"` |
| **item_properties** | `item_id` (PK/FK), `property_id` (PK/FK), `value` | All property values including name/description | `"item1", 1.0, "MacBook Pro"`<br>`"item1", 2.0, "16-inch laptop"`<br>`"item1", 3.0, "16 inches"` |
| **selected_properties** | `url_id` (PK/FK), `property_id` (PK/FK) | Active properties per URL (excludes core) | `1, 3.0` |
### Data Flow
```mermaid
@ -59,6 +60,45 @@ flowchart LR
properties -->|property_id| item_properties
```
### Properties data flow
```mermaid
sequenceDiagram
participant User
participant App as Application
participant Wikidata
User->>App: Enters search
App->>Wikidata: fetch_wikidata_suggestions()
Wikidata-->>App: Return suggestions
App->>User: Show suggestions
User->>App: Selects item
App->>Wikidata: fetch_item_properties()
Wikidata-->>App: Return properties (IDs + values)
App->>Wikidata: fetch_property_labels()
Wikidata-->>App: Return labels
App->>App: Combine labels + properties
App->>User: Show labeled properties
```
## **Docker Deployment**
### **Prerequisites**
- Docker installed on your system
- Docker Compose (usually included with Docker Desktop)
### **Running with Docker**
1. Clone the repository:
```bash
git clone https://forge.ftt.gmbh/ryanmwangi/Compware.git
cd compareware
```
2. Start the container:
```bash
docker-compose up -d
```
3. Access the application at: [http://localhost:3004](http://localhost:3004)
### **Collaboration**
We welcome contributions! Heres how you can help:

11
docker-compose.yml Normal file
View file

@ -0,0 +1,11 @@
services:
app:
build: .
ports:
- "3000:3000"
volumes:
- ./compareware.db:/app/compareware.db
environment:
- RUST_LOG=info
- LEPTOS_ENV=production
restart: unless-stopped

56
dockerfile Normal file
View file

@ -0,0 +1,56 @@
# Build stage
FROM rust:1.83.0-slim-bullseye as builder
# Install essential build tools
RUN apt-get update && \
apt-get install -y \
libsqlite3-dev \
build-essential \
clang \
libssl-dev \
pkg-config \
curl \
cmake \
protobuf-compiler \
&& rm -rf /var/lib/apt/lists/*
# Install Rust toolchain
RUN rustup component add rust-src
# Install cargo-leptos & wasm-bindgen-cli
RUN cargo install cargo-leptos --version 0.2.24 --locked
RUN cargo install wasm-bindgen-cli --version 0.2.99 --locked
# Build application
WORKDIR /app
COPY . .
# Explicitly set WASM target
RUN rustup target add wasm32-unknown-unknown
# Build project
ENV LEPTOS_OUTPUT_NAME="compareware"
# Build with release profile
RUN cargo leptos build --release
# Runtime stage
FROM debian:bullseye-slim
# Install runtime dependencies in Debian
RUN apt-get update && \
apt-get install -y \
libssl-dev \
libsqlite3-0 \
ca-certificates \
&& rm -rf /var/lib/apt/lists/*
# Copy build artifacts
COPY --from=builder /app/target/release/compareware /app/
COPY --from=builder /app/target/site /app/site
COPY assets /app/assets
# Configure container, expose port and set entrypoint
WORKDIR /app
EXPOSE 3000
ENV LEPTOS_SITE_ADDR=0.0.0.0:3000
ENV LEPTOS_SITE_ROOT="site"
CMD ["./compareware"]

View file

@ -141,6 +141,8 @@ pub fn ItemsList(
// Signal to store the fetched property labels
let (property_labels, set_property_labels) = create_signal(HashMap::<String, String>::new());
// State to manage property cache
let (property_cache, set_property_cache) = create_signal(HashMap::<String, HashMap<String, String>>::new());
#[cfg(feature = "ssr")]
fn get_current_url() -> String {
use leptos::use_context;
@ -381,15 +383,31 @@ pub fn ItemsList(
};
//function to fetch properties
async fn fetch_item_properties(wikidata_id: &str) -> HashMap<String, String> {
async fn fetch_item_properties(
wikidata_id: &str,
set_property_labels: WriteSignal<HashMap<String, String>>,
property_cache: ReadSignal<HashMap<String, HashMap<String, String>>>,
set_property_cache: WriteSignal<HashMap<String, HashMap<String, String>>>,
property_labels: ReadSignal<HashMap<String, String>>,
) -> HashMap<String, String> {
// Check cache first
if let Some(cached) = property_cache.get().get(wikidata_id) {
return cached.clone();
}
let sparql_query = format!(
r#"
SELECT ?propLabel ?value ?valueLabel WHERE {{
SELECT ?prop ?propLabel ?value ?valueLabel WHERE {{
wd:{} ?prop ?statement.
?statement ?ps ?value.
?property wikibase:claim ?prop.
?property wikibase:statementProperty ?ps.
SERVICE wikibase:label {{ bd:serviceParam wikibase:language "en". }}
SERVICE wikibase:label {{
bd:serviceParam wikibase:language "en".
?prop rdfs:label ?propLabel.
?value rdfs:label ?valueLabel.
}}
}}
"#,
wikidata_id
@ -408,17 +426,66 @@ pub fn ItemsList(
Ok(response) => {
if let Ok(data) = response.json::<serde_json::Value>().await {
let mut result = HashMap::new();
let mut prop_ids = Vec::new();
// First pass: collect unique property IDs
if let Some(bindings) = data["results"]["bindings"].as_array() {
for binding in bindings {
let prop_label = binding["propLabel"]["value"].as_str().unwrap_or("").to_string();
let prop_label = prop_label.replace("http://www.wikidata.org/prop/", "");
let value_label = binding["valueLabel"]["value"].as_str().unwrap_or("").to_string();
result.insert(prop_label, value_label);
log!("result: {:?}", result);
if let Some(prop) = binding["propLabel"]["value"].as_str() {
let prop_id = prop.replace("http://www.wikidata.org/prop/", "");
if !prop_ids.contains(&prop_id) {
prop_ids.push(prop_id.clone());
}
}
}
}
// Batch fetch missing labels
let existing_labels = property_labels.get();
let missing_ids: Vec<String> = prop_ids
.iter()
.filter(|id| !existing_labels.contains_key(*id))
.cloned()
.collect();
if !missing_ids.is_empty() {
let new_labels = fetch_property_labels(missing_ids).await;
set_property_labels.update(|labels| {
labels.extend(new_labels.clone());
});
}
// Second pass: build results
if let Some(bindings) = data["results"]["bindings"].as_array() {
for binding in bindings {
let prop_label = binding["propLabel"]["value"].as_str().unwrap_or_default();
let value = binding["valueLabel"]["value"]
.as_str()
.or_else(|| binding["value"]["value"].as_str())
.unwrap_or_default();
if let Some(prop_uri) = binding["prop"]["value"].as_str() {
let prop_id = prop_uri.split('/').last().unwrap_or_default().to_string();
result.insert(
prop_id.clone(),
value.to_string()
);
// Update labels if missing
set_property_labels.update(|labels| {
labels.entry(prop_id.clone())
.or_insert(prop_label.to_string());
});
}
}
}
// Update cache
set_property_cache.update(|cache| {
cache.insert(wikidata_id.to_string(), result.clone());
});
result
} else {
HashMap::new()
}
@ -515,11 +582,28 @@ pub fn ItemsList(
let add_property = {
let current_url = Rc::clone(&current_url);
let set_items = set_items.clone();
let set_property_labels = set_property_labels.clone();
let property_cache = property_cache.clone();
let set_property_cache = set_property_cache.clone();
Arc::new(move |property: String| {
// Normalize the property ID
let normalized_property = property.replace("http://www.wikidata.org/prop/", "");
let normalized_property_clone = normalized_property.clone();
// Check if label already exists
if !property_labels.get().contains_key(&normalized_property) {
spawn_local({
let normalized_property = normalized_property.clone();
let set_property_labels = set_property_labels.clone();
async move {
let labels = fetch_property_labels(vec![normalized_property.clone()]).await;
set_property_labels.update(|map| {
map.extend(labels);
});
}
});
}
// Check if property is already selected
if !selected_properties.get().contains_key(&normalized_property) && !normalized_property.is_empty() {
// Add property to selected properties
@ -581,43 +665,45 @@ pub fn ItemsList(
}
});
// Fetch the property label
let property_id = normalized_property.clone();
spawn_local(async move {
let labels = fetch_property_labels(vec![property_id.clone()]).await;
log!("Fetched labels: {:?}", labels);
set_property_labels.update(|labels_map| {
for (key, value) in labels {
log!("Inserting label: {} -> {}", key, value);
labels_map.insert(key, value);
}
});
});
// Use the property label from the property_labels signal
let property_label = property_labels.get().get(&normalized_property).cloned().unwrap_or_else(|| normalized_property.clone());
log!("Added property with label: {}", property_label);
}
});
// Fetch the relevant value for each item and populate the corresponding cells
set_items.update(|items| {
for item in items {
// Initialize property with empty string if it doesn't exist
item.custom_properties.entry(normalized_property.clone())
.or_insert_with(|| "".to_string());
// Only fetch properties if Wikidata ID exists
if let Some(wikidata_id) = &item.wikidata_id {
let wikidata_id = wikidata_id.clone();
let set_items = set_items.clone();
let set_fetched_properties = set_fetched_properties.clone();
let set_property_labels = set_property_labels.clone();
let property_clone = property.clone();
let property_clone = normalized_property.clone();
spawn_local(async move {
let properties = fetch_item_properties(&wikidata_id).await;
// Update fetched properties and property labels
set_fetched_properties.update(|fp| {
fp.insert(wikidata_id.clone(), properties.clone());
});
set_property_labels.update(|pl| {
for (key, value) in properties.iter() {
pl.entry(key.clone()).or_insert_with(|| value.clone());
}
});
let properties = fetch_item_properties(
&wikidata_id,
set_property_labels.clone(),
property_cache.clone(),
set_property_cache.clone(),
property_labels.clone()
).await;
// Update the specific property for this item
if let Some(value) = properties.get(&property_clone) {
set_items.update(|items| {
if let Some(item) = items.iter_mut().find(|item| item.wikidata_id.as_ref().unwrap() == &wikidata_id) {
item.custom_properties.insert(property_clone.clone(), value.clone());
if let Some(item) = items.iter_mut()
.find(|i| i.wikidata_id.as_ref() == Some(&wikidata_id))
{
item.custom_properties.insert(
property_clone.clone(),
value.clone()
);
}
});
}
@ -646,7 +732,7 @@ pub fn ItemsList(
if let Some(wikidata_id) = &item.wikidata_id {
let wikidata_id = wikidata_id.clone();
spawn_local(async move {
let properties = fetch_item_properties(&wikidata_id).await;
let properties = fetch_item_properties(&wikidata_id, set_property_labels.clone(), property_cache.clone(), set_property_cache.clone(), property_labels.clone()).await;
log!("Fetched properties for index {}: {:?}", index, properties);
});
}
@ -795,7 +881,7 @@ pub fn ItemsList(
// Fetch additional properties from Wikidata
let wikidata_id = id.clone();
spawn_local(async move {
let properties = fetch_item_properties(&wikidata_id).await;
let properties = fetch_item_properties(&wikidata_id, set_property_labels.clone(), property_cache.clone(), set_property_cache.clone(), property_labels.clone()).await;
// log!("Fetched properties for Wikidata ID {}: {:?}", wikidata_id, properties);
// Populate the custom properties for the new item
@ -923,17 +1009,21 @@ pub fn ItemsList(
</table>
<div style="margin-bottom: 20px;">
<input type="text" id="new-property" placeholder="Add New Property" list="properties" on:keydown=move |event| {
if event.key() == "Enter"{
if event.key() == "Enter" {
let input_element = event.target().unwrap().dyn_into::<web_sys::HtmlInputElement>().unwrap();
let property = input_element.value();
if !property.is_empty() {
// Extract the coded name from the selected value
let coded_name = property.split(" - ").next().unwrap_or(&property).to_string();
// Add the property using the coded name
add_property(coded_name);
// Clear the input field
let input_value = input_element.value();
// Extract property ID from "Label (P123)" format
let property_id = input_value
.split(" (")
.last()
.and_then(|s| s.strip_suffix(')'))
.unwrap_or(&input_value)
.to_string();
if !property_id.is_empty() {
// Add the property using the extracted ID
add_property(property_id);
input_element.set_value("");
}
}
@ -941,10 +1031,11 @@ pub fn ItemsList(
<datalist id="properties">
{move || {
let property_labels = property_labels.get().clone();
property_labels.into_iter().map(|(property, label)| {
let property_clone = property.clone();
property_labels.into_iter().map(|(property_id, label)| {
view! {
<option value={property}>{ format!("{} - {}", property_clone, label) }</option>
<option value={format!("{} ({})", label, property_id)}>
{ format!("{} ({})", label, property_id) }
</option>
}
}).collect::<Vec<_>>()
}}

637
src/db.rs
View file

@ -1,13 +1,211 @@
#[cfg(feature = "ssr")]
mod db_impl {
use crate::models::item::Item;
use leptos::logging;
use leptos::logging::log;
use rusqlite::{Connection, Error};
use serde::{Deserialize, Serialize};
use std::collections::{HashMap, HashSet};
use std::sync::Arc;
use tokio::sync::Mutex;
use leptos::logging;
use std::collections::{HashMap, HashSet};
use crate::models::item::Item;
use leptos::logging::log;
use uuid::Uuid;
#[cfg(test)]
mod tests {
use super::*;
use tokio::runtime::Runtime;
use uuid::Uuid;
// Helper function to create test database
async fn create_test_db() -> Database {
log!("[TEST] Creating in-memory test database");
let db = Database::new(":memory:").unwrap();
db.create_schema().await.unwrap();
log!("[TEST] Database schema created");
db
}
// Test database schema creation
#[tokio::test]
async fn test_schema_creation() {
log!("[TEST] Starting test_schema_creation");
let db = create_test_db().await;
// Verify tables exist
let conn = db.conn.lock().await;
let mut stmt = conn
.prepare("SELECT name FROM sqlite_master WHERE type='table'")
.unwrap();
let tables: Vec<String> = stmt
.query_map([], |row| row.get(0))
.unwrap()
.collect::<Result<_, _>>()
.unwrap();
assert!(tables.contains(&"urls".to_string()));
assert!(tables.contains(&"items".to_string()));
assert!(tables.contains(&"properties".to_string()));
assert!(tables.contains(&"item_properties".to_string()));
assert!(tables.contains(&"selected_properties".to_string()));
}
// Item Lifecycle Tests
#[tokio::test]
async fn test_full_item_lifecycle() {
log!("[TEST] Starting test_full_item_lifecycle");
let db = create_test_db().await;
let test_url = "https://example.com";
let test_item = Item {
id: Uuid::new_v4().to_string(),
name: "Test Item".into(),
description: "Test Description".into(),
wikidata_id: Some("Q123".into()),
custom_properties: vec![
("price".into(), "100".into()),
("color".into(), "red".into()),
]
.into_iter()
.collect(),
};
// Test insertion
log!("[TEST] Testing item insertion");
db.insert_item_by_url(test_url, &test_item).await.unwrap();
log!("[TEST] Item insertion - PASSED");
// Test retrieval
log!("[TEST] Testing item retrieval");
let items = db.get_items_by_url(test_url).await.unwrap();
assert_eq!(items.len(), 1);
let stored_item = &items[0];
assert_eq!(stored_item.name, test_item.name);
assert_eq!(stored_item.custom_properties.len(), 2);
log!("[TEST] Item retrieval and validation - PASSED");
// Test update
log!("[TEST] Testing item update");
let mut updated_item = test_item.clone();
updated_item.name = "Updated Name".into();
db.insert_item_by_url(test_url, &updated_item)
.await
.unwrap();
// Verify update
let items = db.get_items_by_url(test_url).await.unwrap();
assert_eq!(items[0].name, "Updated Name");
log!("[TEST] Item update - PASSED");
// Test deletion
log!("[TEST] Testing item deletion");
db.delete_item_by_url(test_url, &test_item.id)
.await
.unwrap();
let items = db.get_items_by_url(test_url).await.unwrap();
assert!(items.is_empty());
log!("[TEST] Item deletion - PASSED");
log!("[TEST] test_full_item_lifecycle completed successfully");
}
//URL Management Tests
#[tokio::test]
async fn test_url_management() {
log!("[TEST] Starting test_url_management");
let db = create_test_db().await;
let test_url = "https://test.com";
// Test URL creation
log!("[TEST] Testing URL creation");
let url_id = db.insert_url(test_url).await.unwrap();
assert!(url_id > 0);
log!("[TEST] URL creation - PASSED");
// Test duplicate URL handling
log!("[TEST] Testing duplicate URL handling");
let duplicate_id = db.insert_url(test_url).await.unwrap();
assert_eq!(url_id, duplicate_id);
log!("[TEST] Duplicate URL handling - PASSED");
// Test URL retrieval
log!("[TEST] Testing URL retrieval");
let conn = db.conn.lock().await;
let stored_url: String = conn
.query_row("SELECT url FROM urls WHERE id = ?", [url_id], |row| {
row.get(0)
})
.unwrap();
assert_eq!(stored_url, test_url);
log!("[TEST] URL retrieval - PASSED");
log!("[TEST] test_url_management completed successfully");
}
//property management tests
#[tokio::test]
async fn test_property_operations() {
log!("[TEST] Starting test_property_operations");
let db = create_test_db().await;
let test_url = "https://props.com";
let test_item = Item {
id: Uuid::new_v4().to_string(),
name: "Test Item".into(),
description: "Test Description".into(),
wikidata_id: Some("Q123".into()),
custom_properties: vec![
("price".into(), "100".into()),
("color".into(), "red".into()),
]
.into_iter()
.collect(),
};
// Test property creation
log!("[TEST] Testing property creation");
db.insert_item_by_url(test_url, &test_item).await.unwrap();
// Verify properties stored
let items = db.get_items_by_url(test_url).await.unwrap();
assert_eq!(items[0].custom_properties.len(), 2);
log!("[TEST] Property creation - PASSED");
// Test property deletion
log!("[TEST] Testing property deletion");
db.delete_property_by_url(test_url, "price").await.unwrap();
let items = db.get_items_by_url(test_url).await.unwrap();
assert_eq!(items[0].custom_properties.len(), 1);
assert!(!items[0].custom_properties.contains_key("price"));
log!("[TEST] Property deletion - PASSED");
log!("[TEST] test_property_operations completed successfully");
}
//selected properties test
#[tokio::test]
async fn test_selected_properties() {
log!("[TEST] Starting test_selected_properties");
let db = create_test_db().await;
let test_url = "https://selected.com";
// Add test properties
log!("[TEST] Adding selected properties");
db.add_selected_property(test_url, "price").await.unwrap();
db.add_selected_property(test_url, "weight").await.unwrap();
// Test retrieval
log!("[TEST] Testing property retrieval");
let props = db.get_selected_properties(test_url).await.unwrap();
assert_eq!(props.len(), 2);
assert!(props.contains(&"price".to_string()));
assert!(props.contains(&"weight".to_string()));
log!("[TEST] Property retrieval - PASSED");
// Test duplicate prevention
log!("[TEST] Testing duplicate prevention");
db.add_selected_property(test_url, "price").await.unwrap();
let props = db.get_selected_properties(test_url).await.unwrap();
assert_eq!(props.len(), 2); // No duplicate added
log!("[TEST] Duplicate prevention - PASSED");
log!("[TEST] test_selected_properties completed successfully");
}
}
// Define a struct to represent a database connection
#[derive(Debug)]
@ -19,7 +217,7 @@ mod db_impl {
// Create a new database connection
pub fn new(db_path: &str) -> Result<Self, Error> {
let conn = Connection::open(db_path)?;
logging::log!("Database connection established at: {}", db_path);
logging::log!("Database connection established at: {}", db_path);
Ok(Database {
conn: Arc::new(Mutex::new(conn)),
})
@ -28,15 +226,16 @@ mod db_impl {
// Create the database schema
pub async fn create_schema(&self) -> Result<(), Error> {
let conn = self.conn.lock().await;
// 1. Properties table
conn.execute_batch(
"CREATE TABLE IF NOT EXISTS properties (
id INTEGER PRIMARY KEY,
name TEXT NOT NULL UNIQUE,
global_usage_count INTEGER DEFAULT 0
);"
).map_err(|e| {
);",
)
.map_err(|e| {
eprintln!("Failed creating properties table: {}", e);
e
})?;
@ -48,7 +247,8 @@ mod db_impl {
url TEXT NOT NULL UNIQUE,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
);",
).map_err(|e| {
)
.map_err(|e| {
eprintln!("Failed creating urls table: {}", e);
e
})?;
@ -58,16 +258,35 @@ mod db_impl {
"CREATE TABLE IF NOT EXISTS items (
id TEXT PRIMARY KEY,
url_id INTEGER NOT NULL,
name TEXT NOT NULL,
description TEXT,
wikidata_id TEXT,
item_order INTEGER NOT NULL DEFAULT 0,
FOREIGN KEY (url_id) REFERENCES urls(id) ON DELETE CASCADE
);",
).map_err(|e| {
);
INSERT OR IGNORE INTO properties (name) VALUES
('name'),
('description');",
)
.map_err(|e| {
eprintln!("Failed creating items table: {}", e);
e
})?;
// Check if the global_item_id column exists
let mut stmt = conn.prepare("PRAGMA table_info(items);")?;
let columns: Vec<String> = stmt
.query_map([], |row| row.get(1))? // Column 1 contains the column names
.collect::<Result<_, _>>()?;
if !columns.contains(&"global_item_id".to_string()) {
conn.execute_batch(
"ALTER TABLE items ADD COLUMN global_item_id TEXT;"
)
.map_err(|e| {
eprintln!("Failed adding global_item_id to items table: {}", e);
e
})?;
}
// 4. Table for selected properties
conn.execute_batch(
"CREATE TABLE IF NOT EXISTS selected_properties (
@ -76,34 +295,61 @@ mod db_impl {
PRIMARY KEY (url_id, property_id),
FOREIGN KEY (url_id) REFERENCES urls(id) ON DELETE CASCADE,
FOREIGN KEY (property_id) REFERENCES properties(id) ON DELETE CASCADE
);"
).map_err(|e| {
);",
)
.map_err(|e| {
eprintln!("Failed creating properties table: {}", e);
e
})?;
// 5. Junction table for custom properties
conn.execute_batch(
"CREATE TABLE IF NOT EXISTS item_properties (
item_id TEXT NOT NULL,
global_item_id TEXT NOT NULL,
property_id INTEGER NOT NULL,
value TEXT NOT NULL,
PRIMARY KEY (item_id, property_id),
FOREIGN KEY (item_id) REFERENCES items(id) ON DELETE CASCADE,
PRIMARY KEY (global_item_id, property_id),
FOREIGN KEY (global_item_id) REFERENCES items(global_item_id) ON DELETE CASCADE,
FOREIGN KEY (property_id) REFERENCES properties(id) ON DELETE CASCADE
);"
);",
)
.map_err(|e| {
eprintln!("Failed creating item_properties table: {}", e);
e
})?;
// 6. Junction table for deleted properties
conn.execute_batch(
"CREATE TABLE IF NOT EXISTS deleted_properties (
url_id INTEGER NOT NULL,
global_item_id TEXT NOT NULL,
property_id INTEGER NOT NULL,
PRIMARY KEY (url_id, global_item_id, property_id),
FOREIGN KEY (url_id) REFERENCES urls(id) ON DELETE CASCADE,
FOREIGN KEY (global_item_id) REFERENCES items(global_item_id) ON DELETE CASCADE,
FOREIGN KEY (property_id) REFERENCES properties(id) ON DELETE CASCADE
);",
).map_err(|e| {
eprintln!("Failed creating item_properties table: {}", e);
e
})?;
Ok(())
}
// Insert a new URL into the database
pub async fn insert_url(&self, url: &str) -> Result<i64, Error> {
let conn = self.conn.lock().await;
let mut stmt = conn.prepare("INSERT INTO urls (url) VALUES (?)")?;
let url_id = stmt.insert(&[url])?;
let mut conn = self.conn.lock().await;
let tx = conn.transaction()?;
// Use INSERT OR IGNORE to handle duplicates
tx.execute("INSERT OR IGNORE INTO urls (url) VALUES (?)", [url])?;
// Get the URL ID whether it was inserted or already existed
let url_id =
tx.query_row("SELECT id FROM urls WHERE url = ?", [url], |row| row.get(0))?;
tx.commit()?;
logging::log!("URL inserted: {}", url);
Ok(url_id)
}
@ -117,12 +363,15 @@ mod db_impl {
pub async fn delete_property(&self, property: &str) -> Result<(), Error> {
let conn = self.conn.lock().await;
let query = format!("UPDATE items SET custom_properties = json_remove(custom_properties, '$.{}')", property);
let query = format!(
"UPDATE items SET custom_properties = json_remove(custom_properties, '$.{}')",
property
);
conn.execute(&query, []).map_err(|e| Error::from(e))?;
logging::log!("Property deleted: {}", property);
Ok(())
}
// Retrieve all items from the database
pub async fn get_items(&self) -> Result<Vec<DbItem>, Error> {
let conn = self.conn.lock().await;
@ -146,16 +395,15 @@ mod db_impl {
// Retrieve all items from the database for a specific URL
pub async fn get_items_by_url(&self, url: &str) -> Result<Vec<Item>, Error> {
let conn = self.conn.lock().await;
let url_id: Option<i64> = match conn.query_row(
"SELECT id FROM urls WHERE url = ?",
&[url],
|row| row.get(0)
) {
Ok(id) => Some(id),
Err(rusqlite::Error::QueryReturnedNoRows) => None,
Err(e) => return Err(e),
};
let url_id: Option<i64> =
match conn.query_row("SELECT id FROM urls WHERE url = ?", &[url], |row| {
row.get(0)
}) {
Ok(id) => Some(id),
Err(rusqlite::Error::QueryReturnedNoRows) => None,
Err(e) => return Err(e),
};
let url_id = match url_id {
Some(id) => id,
None => return Ok(Vec::new()), // Return empty list if URL not found
@ -163,56 +411,74 @@ mod db_impl {
log!("Fetching items for URL '{}' (ID: {})", url, url_id);
let mut stmt = conn.prepare(
"SELECT i.id, i.name, i.description, i.wikidata_id,
p.name AS prop_name, ip.value
FROM items i
LEFT JOIN item_properties ip ON i.id = ip.item_id
LEFT JOIN properties p ON ip.property_id = p.id
WHERE i.url_id = ?"
"WITH ordered_items AS (
SELECT
i.id,
i.wikidata_id,
i.item_order,
i.global_item_id
FROM items i
WHERE i.url_id = ?
ORDER BY i.item_order ASC
)
SELECT
oi.id,
oi.wikidata_id,
name_ip.value AS name,
desc_ip.value AS description,
json_group_object(p.name, ip.value) as custom_properties
FROM ordered_items oi
LEFT JOIN item_properties ip
ON oi.global_item_id = ip.global_item_id
AND ip.property_id NOT IN (
SELECT property_id
FROM deleted_properties
WHERE url_id = ? AND global_item_id = oi.global_item_id
)
LEFT JOIN properties p
ON ip.property_id = p.id
LEFT JOIN item_properties name_ip
ON oi.global_item_id = name_ip.global_item_id
AND name_ip.property_id = (SELECT id FROM properties WHERE name = 'name')
LEFT JOIN item_properties desc_ip
ON oi.global_item_id = desc_ip.global_item_id
AND desc_ip.property_id = (SELECT id FROM properties WHERE name = 'description')
GROUP BY oi.id
ORDER BY oi.item_order ASC"
)?;
let mut items: HashMap<String, Item> = HashMap::new();
let rows = stmt.query_map([url_id], |row| {
Ok((
row.get::<_, String>(0)?, // id
row.get::<_, String>(1)?, // name
row.get::<_, String>(2)?, // description
row.get::<_, Option<String>>(3)?, // wikidata_id
row.get::<_, Option<String>>(4)?, // prop_name
row.get::<_, Option<String>>(5)?, // value
))
// Change from HashMap to Vec to preserve order
let rows = stmt.query_map([url_id, url_id], |row| {
let custom_props_json: String = row.get(4)?;
let custom_properties: HashMap<String, String> = serde_json::from_str(&custom_props_json)
.unwrap_or_default();
Ok(Item {
id: row.get(0)?,
name: row.get::<_, Option<String>>(2)?.unwrap_or_default(), // Handle NULL values for name
description: row.get::<_, Option<String>>(3)?.unwrap_or_default(), // Handle NULL values for description
wikidata_id: row.get(1)?,
custom_properties,
})
})?;
let mut items = Vec::new();
for row in rows {
let (id, name, desc, wd_id, prop, val) = row?;
let item = items.entry(id.clone()).or_insert(Item {
id,
name,
description: desc,
wikidata_id: wd_id,
custom_properties: HashMap::new(),
});
if let (Some(p), Some(v)) = (prop, val) {
item.custom_properties.insert(p, v);
}
items.push(row?);
}
Ok(items.into_values().collect())
Ok(items)
}
async fn get_or_create_property(
&self,
tx: &mut rusqlite::Transaction<'_>,
prop: &str
&self,
tx: &mut rusqlite::Transaction<'_>,
prop: &str,
) -> Result<i64, Error> {
match tx.query_row(
"SELECT id FROM properties WHERE name = ?",
[prop],
|row| row.get::<_, i64>(0)
) {
match tx.query_row("SELECT id FROM properties WHERE name = ?", [prop], |row| {
row.get::<_, i64>(0)
}) {
Ok(id) => Ok(id),
Err(rusqlite::Error::QueryReturnedNoRows) => {
tx.execute("INSERT INTO properties (name) VALUES (?)", [prop])?;
@ -225,30 +491,28 @@ mod db_impl {
// Insert a new item into the database for a specific URL
pub async fn insert_item_by_url(&self, url: &str, item: &Item) -> Result<(), Error> {
log!("[DB] Starting insert for URL: {}, Item: {}", url, item.id);
// 1. Check database lock acquisition
let lock_start = std::time::Instant::now();
let mut conn = self.conn.lock().await;
log!("[DB] Lock acquired in {:?}", lock_start.elapsed());
// 2. Transaction handling
log!("[DB] Starting transaction");
let mut tx = conn.transaction().map_err(|e| {
log!("[DB] Transaction start failed: {:?}", e);
e
})?;
// 3. URL handling
log!("[DB] Checking URL existence: {}", url);
let url_id = match tx.query_row(
"SELECT id FROM urls WHERE url = ?",
[url],
|row| row.get::<_, i64>(0)
) {
let url_id = match tx.query_row("SELECT id FROM urls WHERE url = ?", [url], |row| {
row.get::<_, i64>(0)
}) {
Ok(id) => {
log!("[DB] Found existing URL ID: {}", id);
id
},
}
Err(rusqlite::Error::QueryReturnedNoRows) => {
log!("[DB] Inserting new URL");
tx.execute("INSERT INTO urls (url) VALUES (?)", [url])?;
@ -258,72 +522,97 @@ mod db_impl {
}
Err(e) => return Err(e.into()),
};
// 4. Item insertion
let max_order: i32 = tx.query_row(
"SELECT COALESCE(MAX(item_order), 0) FROM items WHERE url_id = ?",
[url_id],
|row| row.get(0),
)?;
let global_item_id = match tx.query_row(
"SELECT ip.global_item_id
FROM item_properties ip
JOIN properties p ON ip.property_id = p.id
WHERE p.name = 'name' AND ip.value = ? LIMIT 1",
[&item.name],
|row| row.get::<_, String>(0),
) {
Ok(id) => id, // Reuse existing global_item_id
Err(rusqlite::Error::QueryReturnedNoRows) => {
let new_id = Uuid::new_v4().to_string(); // Generate a new global_item_id
new_id
}
Err(e) => return Err(e.into()),
};
log!("[DB] Upserting item");
tx.execute(
"INSERT INTO items (id, url_id, name, description, wikidata_id)
"INSERT INTO items (id, url_id, wikidata_id, item_order, global_item_id)
VALUES (?, ?, ?, ?, ?)
ON CONFLICT(id) DO UPDATE SET
url_id = excluded.url_id,
name = excluded.name,
description = excluded.description,
wikidata_id = excluded.wikidata_id",
wikidata_id = excluded.wikidata_id,
global_item_id = excluded.global_item_id",
rusqlite::params![
&item.id,
url_id,
&item.name,
&item.description,
&item.wikidata_id
&item.wikidata_id,
max_order + 1,
&global_item_id
],
)?;
log!("[DB] Item upserted successfully");
// Property handling with enhanced logging
// property handling
let core_properties = vec![
("name", &item.name),
("description", &item.description)
];
for (prop, value) in core_properties.into_iter().chain(
item.custom_properties.iter().map(|(k, v)| (k.as_str(), v))
) {
let prop_id = self.get_or_create_property(&mut tx, prop).await?;
tx.execute(
"INSERT INTO item_properties (global_item_id, property_id, value)
VALUES (?, ?, ?)
ON CONFLICT(global_item_id, property_id) DO UPDATE SET
value = excluded.value",
rusqlite::params![&global_item_id, prop_id, value],
)?;
}
// Property synchronization
log!("[DB] Synchronizing properties for item {}", item.id);
let existing_props = {
// Prepare statement and collect existing properties
let mut stmt = tx.prepare(
"SELECT p.name, ip.value
FROM item_properties ip
JOIN properties p ON ip.property_id = p.id
WHERE ip.item_id = ?"
WHERE ip.global_item_id = ?",
)?;
let mapped_rows = stmt.query_map([&item.id], |row| {
Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
})?;
mapped_rows.collect::<Result<HashMap<String, String>, _>>()?
};
for (prop, value) in &item.custom_properties {
// Update existing or insert new
let prop_id = self.get_or_create_property(&mut tx, prop).await?;
if let Some(existing_value) = existing_props.get(prop) {
if existing_value != value {
log!("[DB] Updating property {} from '{}' to '{}'", prop, existing_value, value);
tx.execute(
"UPDATE item_properties
SET value = ?
WHERE item_id = ?
AND property_id = (SELECT id FROM properties WHERE name = ?)",
rusqlite::params![value, &item.id, prop],
)?;
}
} else {
log!("[DB] Adding new property {}", prop);
tx.execute(
"INSERT INTO item_properties (item_id, property_id, value)
VALUES (?, ?, ?)",
rusqlite::params![&item.id, prop_id, value],
)?;
}
}
// Remove deleted properties
let current_props: HashSet<&str> = item.custom_properties.keys().map(|s| s.as_str()).collect();
// Include core properties in current_props check
let mut current_props: HashSet<&str> = item.custom_properties.keys()
.map(|s| s.as_str())
.collect();
current_props.insert("name");
current_props.insert("description");
// Cleanup with core property protection
for (existing_prop, _) in existing_props {
if !current_props.contains(existing_prop.as_str()) {
if !current_props.contains(existing_prop.as_str())
&& !["name", "description"].contains(&existing_prop.as_str())
{
log!("[DB] Removing deleted property {}", existing_prop);
tx.execute(
"DELETE FROM item_properties
@ -344,18 +633,20 @@ mod db_impl {
let tx = conn.transaction()?;
// Get URL ID
let url_id: i64 = tx.query_row(
"SELECT id FROM urls WHERE url = ?",
[url],
|row| row.get(0)
)?;
let url_id: i64 =
tx.query_row("SELECT id FROM urls WHERE url = ?", [url], |row| row.get(0))?;
// Delete item and properties
tx.execute(
"DELETE FROM items WHERE id = ? AND url_id = ?",
[item_id, &url_id.to_string()],
)?;
tx.execute(
"DELETE FROM item_properties WHERE global_item_id = ?",
[item_id],
)?;
tx.commit()?;
Ok(())
}
@ -364,25 +655,34 @@ mod db_impl {
pub async fn delete_property_by_url(&self, url: &str, property: &str) -> Result<(), Error> {
let mut conn = self.conn.lock().await;
let tx = conn.transaction()?;
// Get URL ID
let url_id: i64 = tx.query_row(
"SELECT id FROM urls WHERE url = ?",
[url],
|row| row.get(0)
let url_id: i64 =
tx.query_row("SELECT id FROM urls WHERE url = ?", [url], |row| row.get(0))?;
// Get property ID
let property_id: i64 = tx.query_row(
"SELECT id FROM properties WHERE name = ?",
[property],
|row| row.get(0),
)?;
// Delete property from all items in this URL
tx.execute(
"DELETE FROM item_properties
WHERE property_id IN (
SELECT id FROM properties WHERE name = ?
)
AND item_id IN (
SELECT id FROM items WHERE url_id = ?
)",
[property, &url_id.to_string()],
)?;
// Get all global_item_ids for this URL
{
let mut stmt = tx.prepare("SELECT global_item_id FROM items WHERE url_id = ?")?;
let global_item_ids: Vec<String> = stmt
.query_map([url_id], |row| row.get(0))?
.collect::<Result<_, _>>()?;
// Insert into deleted_properties for each global_item_id
for global_item_id in global_item_ids {
tx.execute(
"INSERT OR IGNORE INTO deleted_properties (url_id, global_item_id, property_id)
VALUES (?, ?, ?)",
rusqlite::params![url_id, global_item_id, property_id],
)?;
}
}
tx.commit()?;
Ok(())
@ -391,19 +691,20 @@ mod db_impl {
pub async fn add_selected_property(&self, url: &str, property: &str) -> Result<(), Error> {
let mut conn = self.conn.lock().await;
let tx = conn.transaction()?;
// Insert URL if it does not exists
tx.execute("INSERT OR IGNORE INTO urls (url) VALUES (?)", [url])?;
// Get URL ID
let url_id = tx.query_row(
"SELECT id FROM urls WHERE url = ?",
[url],
|row| row.get::<_, i64>(0)
)?;
let url_id = tx.query_row("SELECT id FROM urls WHERE url = ?", [url], |row| {
row.get::<_, i64>(0)
})?;
// Get/Create property
let prop_id = match tx.query_row(
"SELECT id FROM properties WHERE name = ?",
[property],
|row| row.get::<_, i64>(0)
|row| row.get::<_, i64>(0),
) {
Ok(id) => id,
Err(_) => {
@ -411,13 +712,13 @@ mod db_impl {
tx.last_insert_rowid()
}
};
// Insert into selected_properties
tx.execute(
"INSERT OR IGNORE INTO selected_properties (url_id, property_id) VALUES (?, ?)",
[url_id, prop_id]
[url_id, prop_id],
)?;
tx.commit()?;
Ok(())
}
@ -429,34 +730,42 @@ mod db_impl {
FROM selected_properties sp
JOIN properties p ON sp.property_id = p.id
JOIN urls u ON sp.url_id = u.id
WHERE u.url = ?"
WHERE u.url = ?",
)?;
let properties = stmt.query_map([url], |row| row.get(0))?;
properties.collect()
}
// function to log database state
pub async fn debug_dump(&self) -> Result<(), Error> {
let conn = self.conn.lock().await;
log!("[DATABASE DEBUG] URLs:");
let mut stmt = conn.prepare("SELECT id, url FROM urls")?;
let urls = stmt.query_map([], |row| {
Ok(format!("ID: {}, URL: {}", row.get::<_, i64>(0)?, row.get::<_, String>(1)?))
Ok(format!(
"ID: {}, URL: {}",
row.get::<_, i64>(0)?,
row.get::<_, String>(1)?
))
})?;
for url in urls {
log!("[DATABASE DEBUG] {}", url?);
}
log!("[DATABASE DEBUG] Items:");
let mut stmt = conn.prepare("SELECT id, name FROM items")?;
let items = stmt.query_map([], |row| {
Ok(format!("ID: {}, Name: '{}'", row.get::<_, String>(0)?, row.get::<_, String>(1)?))
Ok(format!(
"ID: {}, Name: '{}'",
row.get::<_, String>(0)?,
row.get::<_, String>(1)?
))
})?;
for item in items {
log!("[DATABASE DEBUG] {}", item?);
}
Ok(())
}
}
@ -472,4 +781,4 @@ mod db_impl {
}
#[cfg(feature = "ssr")]
pub use db_impl::{Database, DbItem};
pub use db_impl::{Database, DbItem};