ryanmwangi/Compware
1
0
Fork 0
Code Issues Pull requests Releases Activity

Compare commits

base: ryanmwangi:main
Branches Tags
ryanmwangi:main
ryanmwangi:dev
ryanmwangi:readme-link
ryanmwangi:v0.4.1
ryanmwangi:v0.4.0
ryanmwangi:v0.3.2
ryanmwangi:v0.3.1
ryanmwangi:v0.3.0
ryanmwangi:v0.2.0
ryanmwangi:v0.1.4
ryanmwangi:v0.1.3
ryanmwangi:v0.1.2
ryanmwangi:v0.1.1
ryanmwangi:v0.1.0
ryanmwangi:2412_editable-cells-experiment
..
compare: ryanmwangi:readme-link
Branches Tags
ryanmwangi:dev
ryanmwangi:main
ryanmwangi:readme-link
ryanmwangi:v0.4.1
ryanmwangi:v0.4.0
ryanmwangi:v0.3.2
ryanmwangi:v0.3.1
ryanmwangi:v0.3.0
ryanmwangi:v0.2.0
ryanmwangi:v0.1.4
ryanmwangi:v0.1.3
ryanmwangi:v0.1.2
ryanmwangi:v0.1.1
ryanmwangi:v0.1.0
ryanmwangi:2412_editable-cells-experiment

No commits in common. "main" and "readme-link" have entirely different histories.
main ... readme-lin

4 changed files with 83 additions and 267 deletions
Show stats Expand all files Collapse all files

3
.gitignore vendored
View file

@ -18,5 +18,4 @@ playwright/.cache/
.idea/ .idea/
# Ignore database file # Ignore database file
compareware.db compareware.db
.qodo

22
README.md
View file

@ -59,28 +59,6 @@ flowchart LR
items -->|item_id| item_properties items -->|item_id| item_properties
properties -->|property_id| item_properties properties -->|property_id| item_properties
``` ```
### Properties data flow
```mermaid
sequenceDiagram
participant User
participant App as Application
participant Wikidata
User->>App: Enters search
App->>Wikidata: fetch_wikidata_suggestions()
Wikidata-->>App: Return suggestions
App->>User: Show suggestions
User->>App: Selects item
App->>Wikidata: fetch_item_properties()
Wikidata-->>App: Return properties (IDs + values)
App->>Wikidata: fetch_property_labels()
Wikidata-->>App: Return labels
App->>App: Combine labels + properties
App->>User: Show labeled properties
```
## **Docker Deployment** ## **Docker Deployment**
### **Prerequisites** ### **Prerequisites**

191
src/components/items_list.rs
View file

@ -141,8 +141,6 @@ pub fn ItemsList(
// Signal to store the fetched property labels // Signal to store the fetched property labels
let (property_labels, set_property_labels) = create_signal(HashMap::<String, String>::new()); let (property_labels, set_property_labels) = create_signal(HashMap::<String, String>::new());
// State to manage property cache
let (property_cache, set_property_cache) = create_signal(HashMap::<String, HashMap<String, String>>::new());
#[cfg(feature = "ssr")] #[cfg(feature = "ssr")]
fn get_current_url() -> String { fn get_current_url() -> String {
use leptos::use_context; use leptos::use_context;
@ -383,31 +381,15 @@ pub fn ItemsList(
}; };
//function to fetch properties //function to fetch properties
async fn fetch_item_properties( async fn fetch_item_properties(wikidata_id: &str) -> HashMap<String, String> {
wikidata_id: &str,
set_property_labels: WriteSignal<HashMap<String, String>>,
property_cache: ReadSignal<HashMap<String, HashMap<String, String>>>,
set_property_cache: WriteSignal<HashMap<String, HashMap<String, String>>>,
property_labels: ReadSignal<HashMap<String, String>>,
) -> HashMap<String, String> {
// Check cache first
if let Some(cached) = property_cache.get().get(wikidata_id) {
return cached.clone();
}
let sparql_query = format!( let sparql_query = format!(
r#" r#"
SELECT ?prop ?propLabel ?value ?valueLabel WHERE {{ SELECT ?propLabel ?value ?valueLabel WHERE {{
wd:{} ?prop ?statement. wd:{} ?prop ?statement.
?statement ?ps ?value. ?statement ?ps ?value.
?property wikibase:claim ?prop. ?property wikibase:claim ?prop.
?property wikibase:statementProperty ?ps. ?property wikibase:statementProperty ?ps.
SERVICE wikibase:label {{ SERVICE wikibase:label {{ bd:serviceParam wikibase:language "en". }}
bd:serviceParam wikibase:language "en".
?prop rdfs:label ?propLabel.
?value rdfs:label ?valueLabel.
}}
}} }}
"#, "#,
wikidata_id wikidata_id
@ -426,66 +408,17 @@ pub fn ItemsList(
Ok(response) => { Ok(response) => {
if let Ok(data) = response.json::<serde_json::Value>().await { if let Ok(data) = response.json::<serde_json::Value>().await {
let mut result = HashMap::new(); let mut result = HashMap::new();
let mut prop_ids = Vec::new();
// First pass: collect unique property IDs
if let Some(bindings) = data["results"]["bindings"].as_array() { if let Some(bindings) = data["results"]["bindings"].as_array() {
for binding in bindings { for binding in bindings {
if let Some(prop) = binding["propLabel"]["value"].as_str() { let prop_label = binding["propLabel"]["value"].as_str().unwrap_or("").to_string();
let prop_id = prop.replace("http://www.wikidata.org/prop/", ""); let prop_label = prop_label.replace("http://www.wikidata.org/prop/", "");
if !prop_ids.contains(&prop_id) { let value_label = binding["valueLabel"]["value"].as_str().unwrap_or("").to_string();
prop_ids.push(prop_id.clone()); result.insert(prop_label, value_label);
} log!("result: {:?}", result);
}
} }
} }
// Batch fetch missing labels
let existing_labels = property_labels.get();
let missing_ids: Vec<String> = prop_ids
.iter()
.filter(|id| !existing_labels.contains_key(*id))
.cloned()
.collect();
if !missing_ids.is_empty() {
let new_labels = fetch_property_labels(missing_ids).await;
set_property_labels.update(|labels| {
labels.extend(new_labels.clone());
});
}
// Second pass: build results
if let Some(bindings) = data["results"]["bindings"].as_array() {
for binding in bindings {
let prop_label = binding["propLabel"]["value"].as_str().unwrap_or_default();
let value = binding["valueLabel"]["value"]
.as_str()
.or_else(|| binding["value"]["value"].as_str())
.unwrap_or_default();
if let Some(prop_uri) = binding["prop"]["value"].as_str() {
let prop_id = prop_uri.split('/').last().unwrap_or_default().to_string();
result.insert(
prop_id.clone(),
value.to_string()
);
// Update labels if missing
set_property_labels.update(|labels| {
labels.entry(prop_id.clone())
.or_insert(prop_label.to_string());
});
}
}
}
// Update cache
set_property_cache.update(|cache| {
cache.insert(wikidata_id.to_string(), result.clone());
});
result result
} else { } else {
HashMap::new() HashMap::new()
} }
@ -582,28 +515,11 @@ pub fn ItemsList(
let add_property = { let add_property = {
let current_url = Rc::clone(&current_url); let current_url = Rc::clone(&current_url);
let set_items = set_items.clone(); let set_items = set_items.clone();
let set_property_labels = set_property_labels.clone();
let property_cache = property_cache.clone();
let set_property_cache = set_property_cache.clone();
Arc::new(move |property: String| { Arc::new(move |property: String| {
// Normalize the property ID // Normalize the property ID
let normalized_property = property.replace("http://www.wikidata.org/prop/", ""); let normalized_property = property.replace("http://www.wikidata.org/prop/", "");
let normalized_property_clone = normalized_property.clone(); let normalized_property_clone = normalized_property.clone();
// Check if label already exists
if !property_labels.get().contains_key(&normalized_property) {
spawn_local({
let normalized_property = normalized_property.clone();
let set_property_labels = set_property_labels.clone();
async move {
let labels = fetch_property_labels(vec![normalized_property.clone()]).await;
set_property_labels.update(|map| {
map.extend(labels);
});
}
});
}
// Check if property is already selected // Check if property is already selected
if !selected_properties.get().contains_key(&normalized_property) && !normalized_property.is_empty() { if !selected_properties.get().contains_key(&normalized_property) && !normalized_property.is_empty() {
// Add property to selected properties // Add property to selected properties
@ -665,45 +581,43 @@ pub fn ItemsList(
} }
}); });
// Use the property label from the property_labels signal // Fetch the property label
let property_label = property_labels.get().get(&normalized_property).cloned().unwrap_or_else(|| normalized_property.clone()); let property_id = normalized_property.clone();
log!("Added property with label: {}", property_label); spawn_local(async move {
let labels = fetch_property_labels(vec![property_id.clone()]).await;
log!("Fetched labels: {:?}", labels);
set_property_labels.update(|labels_map| {
for (key, value) in labels {
log!("Inserting label: {} -> {}", key, value);
labels_map.insert(key, value);
}
});
});
} }
}); });
// Fetch the relevant value for each item and populate the corresponding cells // Fetch the relevant value for each item and populate the corresponding cells
set_items.update(|items| { set_items.update(|items| {
for item in items { for item in items {
// Initialize property with empty string if it doesn't exist
item.custom_properties.entry(normalized_property.clone())
.or_insert_with(|| "".to_string());
// Only fetch properties if Wikidata ID exists
if let Some(wikidata_id) = &item.wikidata_id { if let Some(wikidata_id) = &item.wikidata_id {
let wikidata_id = wikidata_id.clone(); let wikidata_id = wikidata_id.clone();
let set_items = set_items.clone();
let set_fetched_properties = set_fetched_properties.clone(); let set_fetched_properties = set_fetched_properties.clone();
let property_clone = normalized_property.clone(); let set_property_labels = set_property_labels.clone();
let property_clone = property.clone();
spawn_local(async move { spawn_local(async move {
let properties = fetch_item_properties( let properties = fetch_item_properties(&wikidata_id).await;
&wikidata_id, // Update fetched properties and property labels
set_property_labels.clone(), set_fetched_properties.update(|fp| {
property_cache.clone(), fp.insert(wikidata_id.clone(), properties.clone());
set_property_cache.clone(), });
property_labels.clone() set_property_labels.update(|pl| {
).await; for (key, value) in properties.iter() {
pl.entry(key.clone()).or_insert_with(|| value.clone());
// Update the specific property for this item }
});
if let Some(value) = properties.get(&property_clone) { if let Some(value) = properties.get(&property_clone) {
set_items.update(|items| { set_items.update(|items| {
if let Some(item) = items.iter_mut() if let Some(item) = items.iter_mut().find(|item| item.wikidata_id.as_ref().unwrap() == &wikidata_id) {
.find(|i| i.wikidata_id.as_ref() == Some(&wikidata_id)) item.custom_properties.insert(property_clone.clone(), value.clone());
{
item.custom_properties.insert(
property_clone.clone(),
value.clone()
);
} }
}); });
} }
@ -732,7 +646,7 @@ pub fn ItemsList(
if let Some(wikidata_id) = &item.wikidata_id { if let Some(wikidata_id) = &item.wikidata_id {
let wikidata_id = wikidata_id.clone(); let wikidata_id = wikidata_id.clone();
spawn_local(async move { spawn_local(async move {
let properties = fetch_item_properties(&wikidata_id, set_property_labels.clone(), property_cache.clone(), set_property_cache.clone(), property_labels.clone()).await; let properties = fetch_item_properties(&wikidata_id).await;
log!("Fetched properties for index {}: {:?}", index, properties); log!("Fetched properties for index {}: {:?}", index, properties);
}); });
} }
@ -881,7 +795,7 @@ pub fn ItemsList(
// Fetch additional properties from Wikidata // Fetch additional properties from Wikidata
let wikidata_id = id.clone(); let wikidata_id = id.clone();
spawn_local(async move { spawn_local(async move {
let properties = fetch_item_properties(&wikidata_id, set_property_labels.clone(), property_cache.clone(), set_property_cache.clone(), property_labels.clone()).await; let properties = fetch_item_properties(&wikidata_id).await;
// log!("Fetched properties for Wikidata ID {}: {:?}", wikidata_id, properties); // log!("Fetched properties for Wikidata ID {}: {:?}", wikidata_id, properties);
// Populate the custom properties for the new item // Populate the custom properties for the new item
@ -1009,21 +923,17 @@ pub fn ItemsList(
</table> </table>
<div style="margin-bottom: 20px;"> <div style="margin-bottom: 20px;">
<input type="text" id="new-property" placeholder="Add New Property" list="properties" on:keydown=move |event| { <input type="text" id="new-property" placeholder="Add New Property" list="properties" on:keydown=move |event| {
if event.key() == "Enter" { if event.key() == "Enter"{
let input_element = event.target().unwrap().dyn_into::<web_sys::HtmlInputElement>().unwrap(); let input_element = event.target().unwrap().dyn_into::<web_sys::HtmlInputElement>().unwrap();
let input_value = input_element.value(); let property = input_element.value();
if !property.is_empty() {
// Extract property ID from "Label (P123)" format // Extract the coded name from the selected value
let property_id = input_value let coded_name = property.split(" - ").next().unwrap_or(&property).to_string();
.split(" (")
.last() // Add the property using the coded name
.and_then(|s| s.strip_suffix(')')) add_property(coded_name);
.unwrap_or(&input_value)
.to_string(); // Clear the input field
if !property_id.is_empty() {
// Add the property using the extracted ID
add_property(property_id);
input_element.set_value(""); input_element.set_value("");
} }
} }
@ -1031,11 +941,10 @@ pub fn ItemsList(
<datalist id="properties"> <datalist id="properties">
{move || { {move || {
let property_labels = property_labels.get().clone(); let property_labels = property_labels.get().clone();
property_labels.into_iter().map(|(property_id, label)| { property_labels.into_iter().map(|(property, label)| {
let property_clone = property.clone();
view! { view! {
<option value={format!("{} ({})", label, property_id)}> <option value={property}>{ format!("{} - {}", property_clone, label) }</option>
{ format!("{} ({})", label, property_id) }
</option>
} }
}).collect::<Vec<_>>() }).collect::<Vec<_>>()
}} }}

134
src/db.rs
View file

@ -8,7 +8,7 @@ mod db_impl {
use std::collections::{HashMap, HashSet}; use std::collections::{HashMap, HashSet};
use std::sync::Arc; use std::sync::Arc;
use tokio::sync::Mutex; use tokio::sync::Mutex;
use uuid::Uuid;
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::*; use super::*;
@ -271,22 +271,6 @@ mod db_impl {
e e
})?; })?;
// Check if the global_item_id column exists
let mut stmt = conn.prepare("PRAGMA table_info(items);")?;
let columns: Vec<String> = stmt
.query_map([], |row| row.get(1))? // Column 1 contains the column names
.collect::<Result<_, _>>()?;
if !columns.contains(&"global_item_id".to_string()) {
conn.execute_batch(
"ALTER TABLE items ADD COLUMN global_item_id TEXT;"
)
.map_err(|e| {
eprintln!("Failed adding global_item_id to items table: {}", e);
e
})?;
}
// 4. Table for selected properties // 4. Table for selected properties
conn.execute_batch( conn.execute_batch(
"CREATE TABLE IF NOT EXISTS selected_properties ( "CREATE TABLE IF NOT EXISTS selected_properties (
@ -305,11 +289,11 @@ mod db_impl {
// 5. Junction table for custom properties // 5. Junction table for custom properties
conn.execute_batch( conn.execute_batch(
"CREATE TABLE IF NOT EXISTS item_properties ( "CREATE TABLE IF NOT EXISTS item_properties (
global_item_id TEXT NOT NULL, item_id TEXT NOT NULL,
property_id INTEGER NOT NULL, property_id INTEGER NOT NULL,
value TEXT NOT NULL, value TEXT NOT NULL,
PRIMARY KEY (global_item_id, property_id), PRIMARY KEY (item_id, property_id),
FOREIGN KEY (global_item_id) REFERENCES items(global_item_id) ON DELETE CASCADE, FOREIGN KEY (item_id) REFERENCES items(id) ON DELETE CASCADE,
FOREIGN KEY (property_id) REFERENCES properties(id) ON DELETE CASCADE FOREIGN KEY (property_id) REFERENCES properties(id) ON DELETE CASCADE
);", );",
) )
@ -317,23 +301,6 @@ mod db_impl {
eprintln!("Failed creating item_properties table: {}", e); eprintln!("Failed creating item_properties table: {}", e);
e e
})?; })?;
// 6. Junction table for deleted properties
conn.execute_batch(
"CREATE TABLE IF NOT EXISTS deleted_properties (
url_id INTEGER NOT NULL,
global_item_id TEXT NOT NULL,
property_id INTEGER NOT NULL,
PRIMARY KEY (url_id, global_item_id, property_id),
FOREIGN KEY (url_id) REFERENCES urls(id) ON DELETE CASCADE,
FOREIGN KEY (global_item_id) REFERENCES items(global_item_id) ON DELETE CASCADE,
FOREIGN KEY (property_id) REFERENCES properties(id) ON DELETE CASCADE
);",
).map_err(|e| {
eprintln!("Failed creating item_properties table: {}", e);
e
})?;
Ok(()) Ok(())
} }
@ -416,8 +383,7 @@ mod db_impl {
SELECT SELECT
i.id, i.id,
i.wikidata_id, i.wikidata_id,
i.item_order, i.item_order
i.global_item_id
FROM items i FROM items i
WHERE i.url_id = ? WHERE i.url_id = ?
ORDER BY i.item_order ASC ORDER BY i.item_order ASC
@ -430,26 +396,25 @@ mod db_impl {
json_group_object(p.name, ip.value) as custom_properties json_group_object(p.name, ip.value) as custom_properties
FROM ordered_items oi FROM ordered_items oi
LEFT JOIN item_properties ip LEFT JOIN item_properties ip
ON oi.global_item_id = ip.global_item_id ON oi.id = ip.item_id
AND ip.property_id NOT IN ( AND ip.property_id NOT IN (
SELECT property_id SELECT id FROM properties WHERE name IN ('name', 'description')
FROM deleted_properties
WHERE url_id = ? AND global_item_id = oi.global_item_id
) )
LEFT JOIN properties p LEFT JOIN properties p
ON ip.property_id = p.id ON ip.property_id = p.id
LEFT JOIN item_properties name_ip LEFT JOIN item_properties name_ip
ON oi.global_item_id = name_ip.global_item_id ON oi.id = name_ip.item_id
AND name_ip.property_id = (SELECT id FROM properties WHERE name = 'name') AND name_ip.property_id = (SELECT id FROM properties WHERE name = 'name')
LEFT JOIN item_properties desc_ip LEFT JOIN item_properties desc_ip
ON oi.global_item_id = desc_ip.global_item_id ON oi.id = desc_ip.item_id
AND desc_ip.property_id = (SELECT id FROM properties WHERE name = 'description') AND desc_ip.property_id = (SELECT id FROM properties WHERE name = 'description')
GROUP BY oi.id GROUP BY oi.id
ORDER BY oi.item_order ASC" ORDER BY oi.item_order ASC"
)?; )?;
// Change from HashMap to Vec to preserve order // Change from HashMap to Vec to preserve order
let rows = stmt.query_map([url_id, url_id], |row| {
let rows = stmt.query_map([url_id], |row| {
let custom_props_json: String = row.get(4)?; let custom_props_json: String = row.get(4)?;
let custom_properties: HashMap<String, String> = serde_json::from_str(&custom_props_json) let custom_properties: HashMap<String, String> = serde_json::from_str(&custom_props_json)
.unwrap_or_default(); .unwrap_or_default();
@ -530,36 +495,18 @@ mod db_impl {
|row| row.get(0), |row| row.get(0),
)?; )?;
let global_item_id = match tx.query_row(
"SELECT ip.global_item_id
FROM item_properties ip
JOIN properties p ON ip.property_id = p.id
WHERE p.name = 'name' AND ip.value = ? LIMIT 1",
[&item.name],
|row| row.get::<_, String>(0),
) {
Ok(id) => id, // Reuse existing global_item_id
Err(rusqlite::Error::QueryReturnedNoRows) => {
let new_id = Uuid::new_v4().to_string(); // Generate a new global_item_id
new_id
}
Err(e) => return Err(e.into()),
};
log!("[DB] Upserting item"); log!("[DB] Upserting item");
tx.execute( tx.execute(
"INSERT INTO items (id, url_id, wikidata_id, item_order, global_item_id) "INSERT INTO items (id, url_id, wikidata_id, item_order)
VALUES (?, ?, ?, ?, ?) VALUES (?, ?, ?, ?)
ON CONFLICT(id) DO UPDATE SET ON CONFLICT(id) DO UPDATE SET
url_id = excluded.url_id, url_id = excluded.url_id,
wikidata_id = excluded.wikidata_id, wikidata_id = excluded.wikidata_id",
global_item_id = excluded.global_item_id",
rusqlite::params![ rusqlite::params![
&item.id, &item.id,
url_id, url_id,
&item.wikidata_id, &item.wikidata_id,
max_order + 1, max_order + 1
&global_item_id
], ],
)?; )?;
log!("[DB] Item upserted successfully"); log!("[DB] Item upserted successfully");
@ -576,11 +523,11 @@ mod db_impl {
let prop_id = self.get_or_create_property(&mut tx, prop).await?; let prop_id = self.get_or_create_property(&mut tx, prop).await?;
tx.execute( tx.execute(
"INSERT INTO item_properties (global_item_id, property_id, value) "INSERT INTO item_properties (item_id, property_id, value)
VALUES (?, ?, ?) VALUES (?, ?, ?)
ON CONFLICT(global_item_id, property_id) DO UPDATE SET ON CONFLICT(item_id, property_id) DO UPDATE SET
value = excluded.value", value = excluded.value",
rusqlite::params![&global_item_id, prop_id, value], rusqlite::params![&item.id, prop_id, value],
)?; )?;
} }
@ -591,7 +538,7 @@ mod db_impl {
"SELECT p.name, ip.value "SELECT p.name, ip.value
FROM item_properties ip FROM item_properties ip
JOIN properties p ON ip.property_id = p.id JOIN properties p ON ip.property_id = p.id
WHERE ip.global_item_id = ?", WHERE ip.item_id = ?",
)?; )?;
let mapped_rows = stmt.query_map([&item.id], |row| { let mapped_rows = stmt.query_map([&item.id], |row| {
@ -641,11 +588,6 @@ mod db_impl {
"DELETE FROM items WHERE id = ? AND url_id = ?", "DELETE FROM items WHERE id = ? AND url_id = ?",
[item_id, &url_id.to_string()], [item_id, &url_id.to_string()],
)?; )?;
tx.execute(
"DELETE FROM item_properties WHERE global_item_id = ?",
[item_id],
)?;
tx.commit()?; tx.commit()?;
Ok(()) Ok(())
@ -655,35 +597,23 @@ mod db_impl {
pub async fn delete_property_by_url(&self, url: &str, property: &str) -> Result<(), Error> { pub async fn delete_property_by_url(&self, url: &str, property: &str) -> Result<(), Error> {
let mut conn = self.conn.lock().await; let mut conn = self.conn.lock().await;
let tx = conn.transaction()?; let tx = conn.transaction()?;
// Get URL ID // Get URL ID
let url_id: i64 = let url_id: i64 =
tx.query_row("SELECT id FROM urls WHERE url = ?", [url], |row| row.get(0))?; tx.query_row("SELECT id FROM urls WHERE url = ?", [url], |row| row.get(0))?;
// Get property ID // Delete property from all items in this URL
let property_id: i64 = tx.query_row( tx.execute(
"SELECT id FROM properties WHERE name = ?", "DELETE FROM item_properties
[property], WHERE property_id IN (
|row| row.get(0), SELECT id FROM properties WHERE name = ?
)
AND item_id IN (
SELECT id FROM items WHERE url_id = ?
)",
[property, &url_id.to_string()],
)?; )?;
// Get all global_item_ids for this URL
{
let mut stmt = tx.prepare("SELECT global_item_id FROM items WHERE url_id = ?")?;
let global_item_ids: Vec<String> = stmt
.query_map([url_id], |row| row.get(0))?
.collect::<Result<_, _>>()?;
// Insert into deleted_properties for each global_item_id
for global_item_id in global_item_ids {
tx.execute(
"INSERT OR IGNORE INTO deleted_properties (url_id, global_item_id, property_id)
VALUES (?, ?, ?)",
rusqlite::params![url_id, global_item_id, property_id],
)?;
}
}
tx.commit()?; tx.commit()?;
Ok(()) Ok(())
} }