feat(api): on-write search reindex after catalogue writes (#17)
Wire best-effort Meilisearch index sync into the admin write paths (create/update/delete/set_fields/set_visibility). Adds SearchClient::sync_object (reindex if the object exists, remove if gone — one uniform path), an optional AppState.search client, and a reindex helper that logs failures via tracing without failing the committed write. Server gains MEILI_URL/MEILI_MASTER_KEY/MEILI_INDEX config; search stays disabled (no-op) when unset. reindex_all remains the recovery path. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -13,12 +13,15 @@ time.workspace = true
|
||||
tower-sessions.workspace = true
|
||||
tower-sessions-sqlx-store.workspace = true
|
||||
sqlx.workspace = true
|
||||
tracing.workspace = true
|
||||
auth = { path = "../auth" }
|
||||
db = { path = "../db" }
|
||||
domain = { path = "../domain" }
|
||||
search = { path = "../search" }
|
||||
|
||||
[dev-dependencies]
|
||||
tokio.workspace = true
|
||||
tower.workspace = true
|
||||
http-body-util.workspace = true
|
||||
serde_json.workspace = true
|
||||
uuid.workspace = true
|
||||
|
||||
@@ -160,6 +160,8 @@ pub(crate) async fn set_visibility(
|
||||
.await
|
||||
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
|
||||
|
||||
crate::reindex(&state, object_id).await;
|
||||
|
||||
Ok(StatusCode::NO_CONTENT)
|
||||
}
|
||||
Err(db::catalog::VisibilityError::ObjectNotFound) => Err(StatusCode::NOT_FOUND),
|
||||
|
||||
@@ -234,6 +234,8 @@ pub(crate) async fn create_object(
|
||||
.await
|
||||
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
|
||||
|
||||
crate::reindex(&state, id).await;
|
||||
|
||||
Ok((
|
||||
StatusCode::CREATED,
|
||||
Json(CreatedObject { id: id.to_string() }),
|
||||
@@ -299,6 +301,8 @@ pub(crate) async fn update_object(
|
||||
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
|
||||
|
||||
if existed {
|
||||
crate::reindex(&state, object_id).await;
|
||||
|
||||
Ok(StatusCode::NO_CONTENT)
|
||||
} else {
|
||||
Err(StatusCode::NOT_FOUND)
|
||||
@@ -339,6 +343,8 @@ pub(crate) async fn delete_object(
|
||||
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
|
||||
|
||||
if existed {
|
||||
crate::reindex(&state, object_id).await;
|
||||
|
||||
Ok(StatusCode::NO_CONTENT)
|
||||
} else {
|
||||
Err(StatusCode::NOT_FOUND)
|
||||
@@ -443,6 +449,8 @@ pub(crate) async fn set_fields(
|
||||
.await
|
||||
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
|
||||
|
||||
crate::reindex(&state, object_id).await;
|
||||
|
||||
Ok(StatusCode::NO_CONTENT)
|
||||
}
|
||||
Err(db::catalog::FieldError::ObjectNotFound) => Err(StatusCode::NOT_FOUND),
|
||||
|
||||
@@ -26,6 +26,23 @@ pub struct AppState {
|
||||
/// Whether the session cookie carries the `Secure` attribute (default true;
|
||||
/// disable only for plain-HTTP self-hosting).
|
||||
pub cookie_secure: bool,
|
||||
/// Search client for on-write index sync. `None` disables indexing (search is a
|
||||
/// best-effort feature; absent when Meilisearch is not configured).
|
||||
pub search: Option<search::SearchClient>,
|
||||
}
|
||||
|
||||
/// Best-effort: keep the search index in step with a catalogue write that has already
|
||||
/// committed. Re-projects and indexes the object, or removes it if it no longer exists.
|
||||
/// Never fails the request — a search outage must not undo a committed write, and
|
||||
/// `reindex_all` is the recovery path. A no-op when search is not configured.
|
||||
pub(crate) async fn reindex(state: &AppState, id: domain::ObjectId) {
|
||||
let Some(search) = &state.search else {
|
||||
return;
|
||||
};
|
||||
|
||||
if let Err(err) = search.sync_object(&state.db, id).await {
|
||||
tracing::error!(?err, object_id = %id, "search reindex after write failed");
|
||||
}
|
||||
}
|
||||
|
||||
/// Build the application router from shared state.
|
||||
|
||||
@@ -12,6 +12,7 @@ fn state(pool: PgPool) -> AppState {
|
||||
db: db::Db::from_pool(pool),
|
||||
app_name: "Test".into(),
|
||||
cookie_secure: false,
|
||||
search: None,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -12,6 +12,7 @@ fn state(pool: PgPool) -> AppState {
|
||||
db: db::Db::from_pool(pool),
|
||||
app_name: "Test".into(),
|
||||
cookie_secure: false,
|
||||
search: None,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -15,6 +15,7 @@ fn state(pool: PgPool) -> AppState {
|
||||
db: db::Db::from_pool(pool),
|
||||
app_name: "Test".into(),
|
||||
cookie_secure: false,
|
||||
search: None,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -10,6 +10,7 @@ fn state(pool: PgPool, app_name: &str) -> AppState {
|
||||
db: db::Db::from_pool(pool),
|
||||
app_name: app_name.to_string(),
|
||||
cookie_secure: false,
|
||||
search: None,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -12,6 +12,7 @@ fn state(pool: PgPool) -> AppState {
|
||||
db: db::Db::from_pool(pool),
|
||||
app_name: "Test".to_string(),
|
||||
cookie_secure: false,
|
||||
search: None,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,137 @@
|
||||
use api::{AppState, build_app, migrate_sessions};
|
||||
use axum::body::Body;
|
||||
use axum::http::{Request, StatusCode, header};
|
||||
use db::users;
|
||||
use domain::{AuditActor, Email, NewUser, ObjectId, Role};
|
||||
use http_body_util::BodyExt;
|
||||
use search::SearchClient;
|
||||
use sqlx::PgPool;
|
||||
use tower::ServiceExt;
|
||||
|
||||
fn meili() -> (String, String) {
|
||||
(
|
||||
std::env::var("MEILI_URL").expect("MEILI_URL must be set"),
|
||||
std::env::var("MEILI_MASTER_KEY").expect("MEILI_MASTER_KEY must be set"),
|
||||
)
|
||||
}
|
||||
|
||||
fn unique_index() -> String {
|
||||
format!("api_reindex_test_{}", uuid::Uuid::new_v4().simple())
|
||||
}
|
||||
|
||||
fn state(pool: PgPool, search: SearchClient) -> AppState {
|
||||
AppState {
|
||||
db: db::Db::from_pool(pool),
|
||||
app_name: "Test".into(),
|
||||
cookie_secure: false,
|
||||
search: Some(search),
|
||||
}
|
||||
}
|
||||
|
||||
async fn seed_user(pool: &PgPool, email: &str, password: &str, role: Role) {
|
||||
let db = db::Db::from_pool(pool.clone());
|
||||
let mut tx = db.pool().begin().await.unwrap();
|
||||
|
||||
users::create_user(
|
||||
&mut tx,
|
||||
AuditActor::System,
|
||||
&NewUser {
|
||||
email: Email::parse(email).unwrap(),
|
||||
password_hash: auth::hash_password(password).unwrap(),
|
||||
role,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
tx.commit().await.unwrap();
|
||||
}
|
||||
|
||||
async fn login(app: &axum::Router, email: &str, password: &str) -> String {
|
||||
let resp = app
|
||||
.clone()
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri("/api/admin/login")
|
||||
.header(header::CONTENT_TYPE, "application/json")
|
||||
.body(Body::from(format!(
|
||||
r#"{{"email":"{email}","password":"{password}"}}"#
|
||||
)))
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(resp.status(), StatusCode::NO_CONTENT);
|
||||
|
||||
resp.headers()
|
||||
.get(header::SET_COOKIE)
|
||||
.unwrap()
|
||||
.to_str()
|
||||
.unwrap()
|
||||
.split(';')
|
||||
.next()
|
||||
.unwrap()
|
||||
.to_owned()
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "../db/migrations")]
|
||||
async fn admin_writes_sync_the_search_index(pool: PgPool) {
|
||||
migrate_sessions(&db::Db::from_pool(pool.clone()))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
seed_user(&pool, "ed@example.com", "pw-editor-123", Role::Editor).await;
|
||||
|
||||
let (url, key) = meili();
|
||||
let search = SearchClient::connect(&url, &key, &unique_index()).unwrap();
|
||||
|
||||
search.ensure_index().await.unwrap();
|
||||
|
||||
// a second handle to the same index, used to observe what the handlers indexed
|
||||
let observer = search.clone();
|
||||
|
||||
let app = build_app(state(pool.clone(), search));
|
||||
let cookie = login(&app, "ed@example.com", "pw-editor-123").await;
|
||||
|
||||
// create via the admin API -> the object is indexed on commit
|
||||
let create = app
|
||||
.clone()
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri("/api/admin/objects")
|
||||
.header(header::COOKIE, &cookie)
|
||||
.header(header::CONTENT_TYPE, "application/json")
|
||||
.body(Body::from(
|
||||
r#"{"object_number":"R-1","object_name":"astrolabe","number_of_objects":1,"visibility":"internal"}"#,
|
||||
))
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(create.status(), StatusCode::CREATED);
|
||||
|
||||
let created: serde_json::Value =
|
||||
serde_json::from_slice(&create.into_body().collect().await.unwrap().to_bytes()).unwrap();
|
||||
let id: ObjectId = created["id"].as_str().unwrap().parse().unwrap();
|
||||
|
||||
assert_eq!(observer.search("astrolabe").await.unwrap(), vec![id]);
|
||||
|
||||
// delete via the admin API -> the object drops out of the index
|
||||
let delete = app
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.method("DELETE")
|
||||
.uri(format!("/api/admin/objects/{id}"))
|
||||
.header(header::COOKIE, &cookie)
|
||||
.body(Body::empty())
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(delete.status(), StatusCode::NO_CONTENT);
|
||||
|
||||
assert!(observer.search("astrolabe").await.unwrap().is_empty());
|
||||
}
|
||||
@@ -40,6 +40,7 @@ pub struct SearchDocument {
|
||||
}
|
||||
|
||||
/// A Meilisearch-backed search client scoped to one index.
|
||||
#[derive(Clone)]
|
||||
pub struct SearchClient {
|
||||
client: meilisearch_sdk::client::Client,
|
||||
index_uid: String,
|
||||
@@ -146,6 +147,23 @@ impl SearchClient {
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Sync a single object's index entry with the database after a catalogue write
|
||||
/// commits: re-project and index it if it still exists, otherwise remove it. This
|
||||
/// is the uniform on-write path for create/update/delete/field/visibility changes —
|
||||
/// a delete (object gone) removes the entry; everything else re-indexes the current
|
||||
/// projection. Best-effort: callers invoke it after the DB transaction commits and
|
||||
/// log (not propagate) any error, since `reindex_all` is the recovery path.
|
||||
pub async fn sync_object(&self, db: &Db, id: ObjectId) -> Result<(), SearchError> {
|
||||
match db::catalog::object_by_id(db.pool(), id).await? {
|
||||
Some(object) => {
|
||||
let document = build_document(db, &object).await?;
|
||||
|
||||
self.index_object(&document).await
|
||||
}
|
||||
None => self.remove_object(id).await,
|
||||
}
|
||||
}
|
||||
|
||||
/// Rebuild the whole index from the database (clears then re-adds all objects).
|
||||
pub async fn reindex_all(&self, db: &Db) -> Result<(), SearchError> {
|
||||
let index = self.client.index(&self.index_uid);
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
use db::{Db, catalog};
|
||||
use domain::{AuditActor, ObjectInput, Visibility};
|
||||
use search::SearchClient;
|
||||
use sqlx::PgPool;
|
||||
|
||||
fn meili() -> (String, String) {
|
||||
(
|
||||
std::env::var("MEILI_URL").expect("MEILI_URL must be set"),
|
||||
std::env::var("MEILI_MASTER_KEY").expect("MEILI_MASTER_KEY must be set"),
|
||||
)
|
||||
}
|
||||
|
||||
fn unique_index() -> String {
|
||||
format!("sync_test_{}", uuid::Uuid::new_v4().simple())
|
||||
}
|
||||
|
||||
fn object(number: &str, name: &str) -> ObjectInput {
|
||||
ObjectInput {
|
||||
object_number: number.into(),
|
||||
object_name: name.into(),
|
||||
number_of_objects: 1,
|
||||
brief_description: None,
|
||||
current_location: None,
|
||||
current_owner: None,
|
||||
recorder: None,
|
||||
recording_date: None,
|
||||
visibility: Visibility::Draft,
|
||||
}
|
||||
}
|
||||
|
||||
#[sqlx::test(migrations = "../db/migrations")]
|
||||
async fn sync_object_indexes_then_removes(pool: PgPool) {
|
||||
let db = Db::from_pool(pool);
|
||||
|
||||
let mut tx = db.pool().begin().await.unwrap();
|
||||
|
||||
let id = catalog::create_object(&mut tx, AuditActor::System, &object("S-1", "lamp"))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
tx.commit().await.unwrap();
|
||||
|
||||
let (url, key) = meili();
|
||||
let client = SearchClient::connect(&url, &key, &unique_index()).unwrap();
|
||||
|
||||
client.ensure_index().await.unwrap();
|
||||
|
||||
// object exists -> sync indexes it
|
||||
client.sync_object(&db, id).await.unwrap();
|
||||
assert_eq!(client.search("lamp").await.unwrap(), vec![id]);
|
||||
|
||||
// object deleted -> sync removes it from the index
|
||||
let mut tx = db.pool().begin().await.unwrap();
|
||||
|
||||
let existed = catalog::delete_object(&mut tx, AuditActor::System, id)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(existed);
|
||||
|
||||
tx.commit().await.unwrap();
|
||||
|
||||
client.sync_object(&db, id).await.unwrap();
|
||||
assert!(client.search("lamp").await.unwrap().is_empty());
|
||||
}
|
||||
@@ -22,6 +22,7 @@ api = { path = "../api" }
|
||||
auth = { path = "../auth" }
|
||||
db = { path = "../db" }
|
||||
domain = { path = "../domain" }
|
||||
search = { path = "../search" }
|
||||
rpassword.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
|
||||
@@ -27,4 +27,19 @@ pub struct Config {
|
||||
default_value_t = true
|
||||
)]
|
||||
pub cookie_secure: bool,
|
||||
|
||||
/// Meilisearch base URL (e.g. `http://localhost:7700`). On-write search indexing
|
||||
/// is enabled only when both this and `--meili-master-key` are set; otherwise
|
||||
/// search is disabled (best-effort feature) and `reindex_all` remains the rebuild
|
||||
/// path.
|
||||
#[arg(long = "meili-url", env = "MEILI_URL")]
|
||||
pub meili_url: Option<String>,
|
||||
|
||||
/// Meilisearch API key (master or a scoped key).
|
||||
#[arg(long = "meili-master-key", env = "MEILI_MASTER_KEY")]
|
||||
pub meili_master_key: Option<String>,
|
||||
|
||||
/// Meilisearch index name for catalogue objects.
|
||||
#[arg(long = "meili-index", env = "MEILI_INDEX", default_value = "objects")]
|
||||
pub meili_index: String,
|
||||
}
|
||||
|
||||
@@ -22,10 +22,34 @@ pub async fn run(config: Config) -> anyhow::Result<()> {
|
||||
.await
|
||||
.context("creating the session store")?;
|
||||
|
||||
let search = match (&config.meili_url, &config.meili_master_key) {
|
||||
(Some(url), Some(key)) => {
|
||||
let client = search::SearchClient::connect(url, key, &config.meili_index)
|
||||
.context("connecting to Meilisearch")?;
|
||||
|
||||
client
|
||||
.ensure_index()
|
||||
.await
|
||||
.context("ensuring the search index exists")?;
|
||||
|
||||
tracing::info!(index = %config.meili_index, "search indexing enabled");
|
||||
|
||||
Some(client)
|
||||
}
|
||||
_ => {
|
||||
tracing::warn!(
|
||||
"MEILI_URL/MEILI_MASTER_KEY not set — search indexing disabled (reindex_all remains the rebuild path)"
|
||||
);
|
||||
|
||||
None
|
||||
}
|
||||
};
|
||||
|
||||
let state = AppState {
|
||||
db,
|
||||
app_name: config.app_name.clone(),
|
||||
cookie_secure: config.cookie_secure,
|
||||
search,
|
||||
};
|
||||
|
||||
let listener = TcpListener::bind(&config.bind_addr)
|
||||
|
||||
@@ -16,6 +16,7 @@ async fn serves_health_live_over_tcp() {
|
||||
db,
|
||||
app_name: "Test".to_string(),
|
||||
cookie_secure: false,
|
||||
search: None,
|
||||
};
|
||||
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
|
||||
Reference in New Issue
Block a user